From 7ef6b65fa960fcedcef8a27b1c3ef8c67ebb2078 Mon Sep 17 00:00:00 2001 From: tirthasheshpatel Date: Tue, 3 Dec 2019 17:49:40 +0530 Subject: [PATCH 0001/4003] BUG: `np.resize` negative shape and subclasses edge case fixes When a negative entry is passed into `np.resize` shape, an error will now be raised reliably. Previously the `-1` semantics of reshape allowed edge cases to pass and return incorrect results. In the corner case of an empty result shape or empty input shape, the result type will now be preserved. --- numpy/core/fromnumeric.py | 30 ++++++++++++++++++------------ numpy/core/tests/test_numeric.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index f09f2a465d0e..d63e410f6c63 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1377,10 +1377,17 @@ def resize(a, new_shape): See Also -------- + np.reshape : Reshape an array without changing the total size. + np.pad : Enlarge and pad an array. + np.repeat: Repeat elements of an array. ndarray.resize : resize an array in-place. Notes ----- + When the total size of the array does not change `~numpy.reshape` should + be used. In most other cases either indexing (to reduce the size) + or padding (to increase the size) may be a more appropriate solution. + Warning: This functionality does **not** consider axes separately, i.e. it does not apply interpolation/extrapolation. It fills the return array with the required number of elements, taken @@ -1404,22 +1411,21 @@ def resize(a, new_shape): """ if isinstance(new_shape, (int, nt.integer)): new_shape = (new_shape,) + a = ravel(a) - Na = len(a) - total_size = um.multiply.reduce(new_shape) - if Na == 0 or total_size == 0: - return mu.zeros(new_shape, a.dtype) - n_copies = int(total_size / Na) - extra = total_size % Na + new_size = 1 + for dim_length in new_shape: + new_size *= dim_length + if dim_length < 0: + raise ValueError('all elements of `new_shape` must be non-negative') - if extra != 0: - n_copies = n_copies + 1 - extra = Na - extra + if a.size == 0 or new_size == 0: + # First case must zero fill. The second would have repeats == 0. + return np.zeros_like(a, shape=new_shape) - a = concatenate((a,) * n_copies) - if extra > 0: - a = a[:-extra] + repeats = -(-new_size // a.size) # ceil division + a = concatenate((a,) * repeats)[:new_size] return reshape(a, new_shape) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index ffebdf64870c..22706a812f58 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -29,6 +29,17 @@ def test_copies(self): Ar3 = np.array([[1, 2, 3], [4, 1, 2], [3, 4, 1], [2, 3, 4]]) assert_equal(np.resize(A, (4, 3)), Ar3) + def test_repeats(self): + A = np.array([1, 2, 3]) + Ar1 = np.array([[1, 2, 3, 1], [2, 3, 1, 2]]) + assert_equal(np.resize(A, (2, 4)), Ar1) + + Ar2 = np.array([[1, 2], [3, 1], [2, 3], [1, 2]]) + assert_equal(np.resize(A, (4, 2)), Ar2) + + Ar3 = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]]) + assert_equal(np.resize(A, (4, 3)), Ar3) + def test_zeroresize(self): A = np.array([[1, 2], [3, 4]]) Ar = np.resize(A, (0,)) @@ -48,6 +59,23 @@ def test_reshape_from_zero(self): assert_array_equal(Ar, np.zeros((2, 1), Ar.dtype)) assert_equal(A.dtype, Ar.dtype) + def test_negative_resize(self): + A = np.arange(0, 10, dtype=np.float32) + new_shape = (-10, -1) + with pytest.raises(ValueError, match=r"negative"): + np.resize(A, new_shape=new_shape) + + def test_subclass(self): + class MyArray(np.ndarray): + __array_priority__ = 1. + + my_arr = np.array([1]).view(MyArray) + assert type(np.resize(my_arr, 5)) is MyArray + assert type(np.resize(my_arr, 0)) is MyArray + + my_arr = np.array([]).view(MyArray) + assert type(np.resize(my_arr, 5)) is MyArray + class TestNonarrayArgs(object): # check that non-array arguments to functions wrap them in arrays From 1809a0fded50a877b20e1d4b2e2a6583dc3158d7 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 31 Jan 2020 00:16:15 -0800 Subject: [PATCH 0002/4003] API: Create Preliminary DTypeMeta class and np.dtype subclasses This makes all dtypes being subclasses of a: class DType(np.dtype): is_abstract = False is_flexible = False % True for void, strings, and datetimes singleton = prototype_dtype_instance # other information defined on the singleton currently subclass of np.dtype. Right now this happens by replacing the Type field of of the singleton instance. This is valid for types/objects not created in python (not HeapTypes). It is necessary to have this functionality for continuing support of user defined legacy dtypes. --- numpy/core/code_generators/genapi.py | 21 +- .../code_generators/generate_numpy_api.py | 4 +- numpy/core/code_generators/numpy_api.py | 4 +- numpy/core/include/numpy/ndarraytypes.h | 78 ++++++ numpy/core/setup.py | 2 + numpy/core/src/multiarray/arraytypes.c.src | 12 + numpy/core/src/multiarray/descriptor.c | 27 +- numpy/core/src/multiarray/dtypemeta.c | 245 ++++++++++++++++++ numpy/core/src/multiarray/dtypemeta.h | 7 + numpy/core/src/multiarray/multiarraymodule.c | 14 +- numpy/core/src/multiarray/usertypes.c | 6 + 11 files changed, 405 insertions(+), 15 deletions(-) create mode 100644 numpy/core/src/multiarray/dtypemeta.c create mode 100644 numpy/core/src/multiarray/dtypemeta.h diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 88dc2d90a4d3..d88772bdcb96 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -37,6 +37,7 @@ join('multiarray', 'datetime_busdaycal.c'), join('multiarray', 'datetime_strings.c'), join('multiarray', 'descriptor.c'), + join('multiarray', 'dtypemeta.c'), join('multiarray', 'einsum.c.src'), join('multiarray', 'flagsobject.c'), join('multiarray', 'getset.c'), @@ -309,11 +310,13 @@ def write_file(filename, data): # Those *Api classes instances know how to output strings for the generated code class TypeApi: - def __init__(self, name, index, ptr_cast, api_name): + def __init__(self, name, index, ptr_cast, api_name, internal_type=None): self.index = index self.name = name self.ptr_cast = ptr_cast self.api_name = api_name + # The type used internally, if None, same as exported (ptr_cast) + self.internal_type = internal_type def define_from_array_api_string(self): return "#define %s (*(%s *)%s[%d])" % (self.name, @@ -325,9 +328,19 @@ def array_api_define(self): return " (void *) &%s" % self.name def internal_define(self): - astr = """\ -extern NPY_NO_EXPORT PyTypeObject %(type)s; -""" % {'type': self.name} + if self.internal_type is None: + return f"extern NPY_NO_EXPORT {self.ptr_cast} {self.name};\n" + + # If we are here, we need to define a larger struct internally, which + # the type can be cast safely. But we want to normally use the original + # type, so name mangle: + mangled_name = f"{self.name}Full" + astr = ( + # Create the mangled name: + f"extern NPY_NO_EXPORT {self.internal_type} {mangled_name};\n" + # And define the name as: (*(type *)(&mangled_name)) + f"#define {self.name} (*({self.ptr_cast} *)(&{mangled_name}))\n" + ) return astr class GlobalVarApi: diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index fe21bc543d58..7997135bb07a 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -201,7 +201,9 @@ def do_generate_api(targets, sources): for name, val in types_api.items(): index = val[0] - multiarray_api_dict[name] = TypeApi(name, index, 'PyTypeObject', api_name) + internal_type = None if len(val) == 1 else val[1] + multiarray_api_dict[name] = TypeApi( + name, index, 'PyTypeObject', api_name, internal_type) if len(multiarray_api_dict) != len(multiarray_api_index): keys_dict = set(multiarray_api_dict.keys()) diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 916fb537e863..fbd3233680fa 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -30,7 +30,9 @@ multiarray_types_api = { 'PyBigArray_Type': (1,), 'PyArray_Type': (2,), - 'PyArrayDescr_Type': (3,), + # Internally, PyArrayDescr_Type is a PyArray_DTypeMeta, + # the following also defines PyArrayDescr_TypeFull (Full appended) + 'PyArrayDescr_Type': (3, "PyArray_DTypeMeta"), 'PyArrayFlags_Type': (4,), 'PyArrayIter_Type': (5,), 'PyArrayMultiIter_Type': (6,), diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index bec6fcf309c3..db7d7bb30614 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1810,6 +1810,84 @@ typedef struct { typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, void *user_data); + +/* + * PyArray_DTypeMeta related definitions. + * + * As of now, this API is preliminary and will be extended as necessary. + */ +#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD + /* + * The Structures defined in this block are considered private API and + * may change without warning! + */ + /* TODO: Make this definition public in the API, as soon as its settled */ + NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type; + + /* + * While NumPy DTypes would not need to be heap types the plan is to + * make DTypes available in Python at which point we will probably want + * them to be. + * Since we also wish to add fields to the DType class, this looks like + * a typical instance definition, but with PyHeapTypeObject instead of + * only the PyObject_HEAD. + * This must only be exposed very extremely careful consideration, since + * it is a fairly complex construct which may be better to allow + * refactoring of. + */ + typedef struct _PyArray_DTypeMeta { + PyHeapTypeObject super; + + /* + * TODO: Update above comment as necessary. + * Most DTypes will have a singleton default instance, for the flexible + * legacy DTypes (bytes, string, void, datetime) this may be a pointer + * to the *prototype* instance? + */ + PyArray_Descr *singleton; + /* + * Is this DType created using the old API? + * TODO: For now this mostly exists for possible assertions, + * we may not need it. + */ + npy_bool is_legacy; + + /* + * The following fields replicate the most important dtype information. + * In the legacy implementation most of these are stored in the + * PyArray_Descr struct. + */ + /* The type object of the scalar instances (may be NULL?) */ + PyTypeObject *scalar_type; + /* kind for this type */ + char kind; + /* unique-character representing this type */ + char type; + /* flags describing data type */ + char flags; + npy_bool is_flexible; + /* whether the DType can be instantiated (i.e. np.dtype cannot) */ + npy_bool is_abstract; + /* number representing this type */ + int type_num; + /* + * itemsize of this type, -1 if variable. Use npy_intp, even though it + * is effectively limited to int due to other public API. + */ + npy_intp itemsize; + /* + * Point to the original ArrFuncs. + * TODO: If we wanted to do, we could make a copy to detect changes. + * However, I doubt that is necessary. + */ + PyArray_ArrFuncs *f; + } PyArray_DTypeMeta; + + #define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr)) + +#endif /* NPY_INTERNAL_BUILD */ + + /* * Use the keyword NPY_DEPRECATED_INCLUDES to ensure that the header files * npy_*_*_deprecated_api.h are only included from here and nowhere else. diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 15e732614b6e..90995aabe5b2 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -783,6 +783,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'conversion_utils.h'), join('src', 'multiarray', 'ctors.h'), join('src', 'multiarray', 'descriptor.h'), + join('src', 'multiarray', 'dtypemeta.h'), join('src', 'multiarray', 'dragon4.h'), join('src', 'multiarray', 'getset.h'), join('src', 'multiarray', 'hashdescr.h'), @@ -841,6 +842,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'datetime_busday.c'), join('src', 'multiarray', 'datetime_busdaycal.c'), join('src', 'multiarray', 'descriptor.c'), + join('src', 'multiarray', 'dtypemeta.c'), join('src', 'multiarray', 'dragon4.c'), join('src', 'multiarray', 'dtype_transfer.c'), join('src', 'multiarray', 'einsum.c.src'), diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 5e07f0df4d54..227706fc6f6c 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -20,6 +20,7 @@ #include "npy_sort.h" #include "common.h" #include "ctors.h" +#include "dtypemeta.h" #include "lowlevel_strided_loops.h" #include "usertypes.h" #include "_datetime.h" @@ -4358,6 +4359,17 @@ set_typeinfo(PyObject *dict) PyArray_Descr *dtype; PyObject *cobj, *key; + /* + * Override the base class for all types, eventually all of this logic + * should be defined on the class and inherited to the scalar. + * (NPY_HALF is the largest builtin one.) + */ + for (i = 0; i <= NPY_HALF; i++) { + if (dtypemeta_wrap_legacy_descriptor(_builtin_descrs[i]) < 0) { + return -1; + } + } + /* * Add cast functions for the new types */ diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index b26a26abf6ae..5c2c4a12cbd9 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1744,7 +1744,7 @@ _convert_from_str(PyObject *obj, int align) NPY_NO_EXPORT PyArray_Descr * PyArray_DescrNew(PyArray_Descr *base) { - PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, &PyArrayDescr_Type); + PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, Py_TYPE(base)); if (newdescr == NULL) { return NULL; @@ -2261,9 +2261,16 @@ static PyGetSetDef arraydescr_getsets[] = { }; static PyObject * -arraydescr_new(PyTypeObject *NPY_UNUSED(subtype), +arraydescr_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) { + if (subtype != &PyArrayDescr_Type) { + /* The DTypeMeta class should prevent this from happening. */ + PyErr_Format(PyExc_SystemError, + "'%S' must not inherit np.dtype.__new__().", subtype); + return NULL; + } + PyObject *odescr, *metadata=NULL; PyArray_Descr *descr, *conv; npy_bool align = NPY_FALSE; @@ -2334,6 +2341,7 @@ arraydescr_new(PyTypeObject *NPY_UNUSED(subtype), return (PyObject *)conv; } + /* * Return a tuple of * (cleaned metadata dictionary, tuple with (str, num)) @@ -3456,21 +3464,30 @@ static PyMappingMethods descr_as_mapping = { /****************** End of Mapping Protocol ******************************/ -NPY_NO_EXPORT PyTypeObject PyArrayDescr_Type = { + +/* + * NOTE: Since this is a MetaClass, the name has Full appended here, the + * correct name of the type is PyArrayDescr_Type. + */ +NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = {{{ PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "numpy.dtype", .tp_basicsize = sizeof(PyArray_Descr), - /* methods */ .tp_dealloc = (destructor)arraydescr_dealloc, .tp_repr = (reprfunc)arraydescr_repr, .tp_as_number = &descr_as_number, .tp_as_sequence = &descr_as_sequence, .tp_as_mapping = &descr_as_mapping, .tp_str = (reprfunc)arraydescr_str, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_richcompare = (richcmpfunc)arraydescr_richcompare, .tp_methods = arraydescr_methods, .tp_members = arraydescr_members, .tp_getset = arraydescr_getsets, .tp_new = arraydescr_new, + },}, + .type_num = -1, + .kind = '\0', + .is_abstract = 1, + .is_flexible = 0, }; diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c new file mode 100644 index 000000000000..832f36215534 --- /dev/null +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -0,0 +1,245 @@ +/* Array Descr Object */ + +#define PY_SSIZE_T_CLEAN +#include +#include +#include "structmember.h" + + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE +#include "numpy/arrayobject.h" +#include "numpy/arrayscalars.h" + +#include "npy_config.h" +#include "npy_ctypes.h" +#include "npy_pycompat.h" + +#include "_datetime.h" +#include "common.h" +#include "alloc.h" +#include "assert.h" + +#include "dtypemeta.h" +#include "convert_datatype.h" + + +static void +dtypemeta_dealloc(PyArray_DTypeMeta *self) { + /* + * PyType_Type asserts Py_TPFLAGS_HEAPTYPE as well. Do not rely on + * a python debug build though. + */ + assert(((PyTypeObject *)self)->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_XDECREF(self->scalar_type); + PyType_Type.tp_dealloc((PyObject *) self); +} + +static PyObject * +dtypemeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyErr_SetString(PyExc_TypeError, + "Preliminary-API: Cannot subclass DType."); + return NULL; +} + +static int +dtypemeta_init(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyErr_SetString(PyExc_TypeError, + "Preliminary-API: Cannot initialize DType class."); + return -1; +} + +/** + * tp_is_gc slot of Python types. This is implemented only for documentation + * purposes to indicate and document the subtleties involved. + * + * Python Type objects are either statically created (typical C-Extension type) + * or HeapTypes (typically created in Python). + * HeapTypes have the Py_TPFLAGS_HEAPTYPE flag, and are garbage collected, our + * DTypeMeta instances (`np.dtype` and its subclasses) *may* be HeapTypes + * if the Py_TPFLAGS_HEAPTYPE flag is set (they are created from Python). + * They are not for legacy DTypes or np.dtype itself. + * + * @param self + * @return nonzero if the object is garbage collected + */ +static NPY_INLINE int +dtypemeta_is_gc(PyObject *dtype_class) +{ + return PyType_Type.tp_is_gc(dtype_class); +} + + +static int +dtypemeta_traverse(PyArray_DTypeMeta *type, visitproc visit, void *arg) +{ + /* + * We have to traverse the base class (if it is a HeapType). + * PyType_Type will handle this logic for us. + * TODO: In the future, we may have to VISIT some python objects held, + * however, only if we are a Py_TPFLAGS_HEAPTYPE. + */ + assert(!type->is_legacy && (PyTypeObject *)type != &PyArrayDescr_Type); + Py_VISIT(type->singleton); + Py_VISIT(type->scalar_type); + return PyType_Type.tp_traverse((PyObject *)type, visit, arg); +} + + +static PyObject * +legacy_dtype_default_new(PyArray_DTypeMeta *self, + PyObject *args, PyObject *kwargs) +{ + /* TODO: This should allow endianess and possibly metadata */ + if (self->is_flexible) { + /* reject flexible ones since we would need to get unit, etc. info */ + PyErr_Format(PyExc_TypeError, + "Preliminary-API: Flexible legacy DType '%S' can only be " + "instantiated using `np.dtype(...)`", self); + return NULL; + } + + if (PyTuple_GET_SIZE(args) != 0) { + PyErr_Format(PyExc_TypeError, + "currently only the no-argument instantiation is supported; " + "use `np.dtype` instead."); + return NULL; + } + Py_INCREF(self->singleton); + return (PyObject *)self->singleton; +} + +/** + * This function takes a PyArray_Descr and replaces its base class with + * a newly created dtype subclass (DTypeMeta instances). + * There are some subtleties that need to be remembered when doing this, + * first for the class objects itself it could be either a HeapType or not. + * Since we are defining the DType from C, we will not make it a HeapType, + * thus making it identical to a typical *static* type (except that we + * malloc it). We could do it the other way, but there seems no reason to + * do so. + * + * The DType instances (the actual dtypes or descriptors), are based on + * prototypes which are passed in. These should not be garbage collected + * and thus Py_TPFLAGS_HAVE_GC is not set. (We could allow this, but than + * would have to allocate a new object, since the GC needs information before + * the actual struct). + * + * The above is the reason why we should works exactly like we would for a + * static type here. + * Otherwise, we blurry the lines between C-defined extension classes + * and Python subclasses. e.g. `class MyInt(int): pass` is very different + * from our `class Float64(np.dtype): pass`, because the latter should not + * be a HeapType and its instances should be exact PyArray_Descr structs. + * + * @param descr The descriptor that should be wrapped. + * @param name The name for the DType, if NULL the type character is used. + * + * @returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) +{ + if (Py_TYPE(descr) != &PyArrayDescr_Type) { + PyErr_Format(PyExc_RuntimeError, + "During creation/wrapping of legacy DType, the original class " + "was not PyArrayDescr_Type (it is replaced in this step)."); + return -1; + } + + /* + * Note: we have no intention of cleaning this up, since this behaves + * identical to static type definition (see comment above). + * This is much cleaner for the legacy API, in the new API both ways + * should be possible. + * In particular our own DTypes can be true static declarations so that + * this function is only needed for legacy user dtypes. + */ + char *tp_name = PyDataMem_NEW(100); + snprintf(tp_name, 100, "numpy.dtype[%s]", + descr->typeobj->tp_name); + + PyArray_DTypeMeta *dtype_class = PyDataMem_NEW(sizeof(PyArray_DTypeMeta)); + if (dtype_class == NULL) { + return -1; + } + /* + * Initialize the struct fields similar to static code: + */ + /* Copy in np.dtype since it shares most things */ + memcpy(dtype_class, &PyArrayDescr_Type, sizeof(PyArray_DTypeMeta)); + + /* Fix name, base, and __new__*/ + ((PyTypeObject *)dtype_class)->tp_name = tp_name; + ((PyTypeObject *)dtype_class)->tp_base = &PyArrayDescr_Type; + ((PyTypeObject *)dtype_class)->tp_new = (newfunc)legacy_dtype_default_new; + + /* Let python finish the initialization (probably unnecessary) */ + if (PyType_Ready((PyTypeObject *)dtype_class) < 0) { + return -1; + } + + /* np.dtype is not a concrete DType, this one is */ + dtype_class->is_abstract = NPY_FALSE; + + Py_INCREF(descr); /* descr is a singleton that must survive, ensure. */ + dtype_class->singleton = descr; + Py_INCREF(descr->typeobj); + dtype_class->scalar_type = descr->typeobj; + dtype_class->is_legacy = NPY_TRUE; + dtype_class->type_num = descr->type_num; + dtype_class->type = descr->type; + dtype_class->f = descr->f; + dtype_class->kind = descr->kind; + dtype_class->itemsize = descr->elsize; + + if (PyTypeNum_ISDATETIME(descr->type_num)) { + /* Datetimes are flexible, but were not considered previously */ + dtype_class->is_flexible = NPY_TRUE; + } + if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { + dtype_class->is_flexible = NPY_TRUE; + dtype_class->itemsize = -1; /* itemsize is not fixed */ + } + + /* Finally, replace the current class of the descr */ + Py_TYPE(descr) = (PyTypeObject *)dtype_class; + + return 0; +} + + +/* + * Simple exposed information, defined for each DType (class). This is + * preliminary (the flags should also return bools). + */ +static PyMemberDef dtypemeta_members[] = { + {"_abstract", + T_BYTE, offsetof(PyArray_DTypeMeta, is_abstract), READONLY, NULL}, + {"type", + T_OBJECT, offsetof(PyArray_DTypeMeta, scalar_type), READONLY, NULL}, + {"_flexible", + T_BYTE, offsetof(PyArray_DTypeMeta, is_flexible), READONLY, NULL}, + {NULL, 0, 0, 0, NULL}, +}; + + +NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "numpy._DTypeMeta", + .tp_basicsize = sizeof(PyArray_DTypeMeta), + /* methods */ + .tp_dealloc = (destructor)dtypemeta_dealloc, + /* Types are garbage collected (see dtypemeta_is_gc documentation) */ + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", + .tp_members = dtypemeta_members, + .tp_base = &PyType_Type, + .tp_init = (initproc)dtypemeta_init, + .tp_new = dtypemeta_new, + .tp_is_gc = dtypemeta_is_gc, + .tp_traverse = (traverseproc)dtypemeta_traverse, +}; + diff --git a/numpy/core/src/multiarray/dtypemeta.h b/numpy/core/src/multiarray/dtypemeta.h new file mode 100644 index 000000000000..97152d1ada8c --- /dev/null +++ b/numpy/core/src/multiarray/dtypemeta.h @@ -0,0 +1,7 @@ +#ifndef _NPY_DTYPEMETA_H +#define _NPY_DTYPEMETA_H + +NPY_NO_EXPORT int +dtypemeta_wrap_legacy_descriptor(PyArray_Descr *dtypem); + +#endif /*_NPY_DTYPEMETA_H */ diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 7792fcdcbf88..af0230090d44 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4440,6 +4440,16 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { if (set_matmul_flags(d) < 0) { goto err; } + + if (PyType_Ready(&PyArrayDTypeMeta_Type) < 0) { + goto err; + } + + PyArrayDescr_Type.tp_hash = PyArray_DescrHash; + if (PyType_Ready(&PyArrayDescr_Type) < 0) { + goto err; + } + initialize_casting_tables(); initialize_numeric_types(); if (initscalarmath(m) < 0) { @@ -4473,10 +4483,6 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } - PyArrayDescr_Type.tp_hash = PyArray_DescrHash; - if (PyType_Ready(&PyArrayDescr_Type) < 0) { - goto err; - } if (PyType_Ready(&PyArrayFlags_Type) < 0) { goto err; } diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 997467b4d677..bc320138d5b1 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -37,6 +37,7 @@ maintainer email: oliphant.travis@ieee.org #include "npy_pycompat.h" #include "usertypes.h" +#include "dtypemeta.h" NPY_NO_EXPORT PyArray_Descr **userdescrs=NULL; @@ -226,6 +227,11 @@ PyArray_RegisterDataType(PyArray_Descr *descr) return -1; } userdescrs[NPY_NUMUSERTYPES++] = descr; + + if (dtypemeta_wrap_legacy_descriptor(descr) < 0) { + return -1; + } + return typenum; } From 4a080dc1240f89cf5411930e3413b9bafe19f5e4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 4 Feb 2020 16:33:15 -0800 Subject: [PATCH 0003/4003] Possible initialization safety fixups and fix API hash. (currently no public API changes, but the hash changed). The type safety may not be important as such, but there used to be issues, at least with MSVC about this, and clang warns. --- numpy/core/code_generators/cversions.txt | 3 +++ numpy/core/src/multiarray/descriptor.c | 1 + numpy/core/src/multiarray/dtypemeta.c | 3 +-- numpy/core/src/multiarray/multiarraymodule.c | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 5daa52d79019..9e2269f87e72 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -51,3 +51,6 @@ # Version 13 (NumPy 1.18) No change. # Version 13 (NumPy 1.19) No change. 0x0000000d = 5b0e8bbded00b166125974fc71e80a33 + +# Version 14 (NumPy 1.19) DType related API additions +0x0000000e = 17a0f366e55ec05e5c5c149123478452 diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 5c2c4a12cbd9..e89c4f6382b7 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -3470,6 +3470,7 @@ static PyMappingMethods descr_as_mapping = { * correct name of the type is PyArrayDescr_Type. */ NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = {{{ + /* The NULL represents `type`, this is set to DTypeMeta at import time */ PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "numpy.dtype", .tp_basicsize = sizeof(PyArray_Descr), diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 832f36215534..5b02396fd3c5 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -230,13 +230,12 @@ NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "numpy._DTypeMeta", .tp_basicsize = sizeof(PyArray_DTypeMeta), - /* methods */ .tp_dealloc = (destructor)dtypemeta_dealloc, /* Types are garbage collected (see dtypemeta_is_gc documentation) */ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", .tp_members = dtypemeta_members, - .tp_base = &PyType_Type, + .tp_base = NULL, /* set to PyType_Type at import time */ .tp_init = (initproc)dtypemeta_init, .tp_new = dtypemeta_new, .tp_is_gc = dtypemeta_is_gc, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index af0230090d44..6423b511a3f3 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4441,11 +4441,13 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } + PyArrayDTypeMeta_Type.tp_base = &PyType_Type; if (PyType_Ready(&PyArrayDTypeMeta_Type) < 0) { goto err; } PyArrayDescr_Type.tp_hash = PyArray_DescrHash; + Py_TYPE(&PyArrayDescr_Type) = &PyArrayDTypeMeta_Type; if (PyType_Ready(&PyArrayDescr_Type) < 0) { goto err; } From 5ced904f29ffb3bdf9ab6e75d9c722d3b1c7f1e1 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 5 Feb 2020 17:36:23 -0800 Subject: [PATCH 0004/4003] Small fixups/comments --- numpy/core/setup_common.py | 3 ++- numpy/core/src/multiarray/dtypemeta.c | 32 +++++++++++---------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index 63c4a76a9bc7..72b59f9ae7f0 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -40,7 +40,8 @@ # 0x0000000c - 1.14.x # 0x0000000c - 1.15.x # 0x0000000d - 1.16.x -C_API_VERSION = 0x0000000d +# 0x0000000e - 1.19.x +C_API_VERSION = 0x0000000e class MismatchCAPIWarning(Warning): pass diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 5b02396fd3c5..b18ef0c3a08c 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -2,26 +2,15 @@ #define PY_SSIZE_T_CLEAN #include -#include #include "structmember.h" - +#include "assert.h" #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE -#include "numpy/arrayobject.h" -#include "numpy/arrayscalars.h" - -#include "npy_config.h" -#include "npy_ctypes.h" +#include #include "npy_pycompat.h" -#include "_datetime.h" -#include "common.h" -#include "alloc.h" -#include "assert.h" - #include "dtypemeta.h" -#include "convert_datatype.h" static void @@ -150,19 +139,24 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) } /* - * Note: we have no intention of cleaning this up, since this behaves - * identical to static type definition (see comment above). - * This is much cleaner for the legacy API, in the new API both ways - * should be possible. - * In particular our own DTypes can be true static declarations so that - * this function is only needed for legacy user dtypes. + * Note: we have no intention of freeing the memory again since this + * behaves identically to static type definition (see comment above). + * This is much cleaner for the legacy API, in the new API both static + * and heap types are possible. + * In particular our own DTypes can be true static declarations. + * However, this function remains necessary for legacy user dtypes. */ char *tp_name = PyDataMem_NEW(100); + if (tp_name == NULL) { + PyErr_NoMemory(); + return -1; + } snprintf(tp_name, 100, "numpy.dtype[%s]", descr->typeobj->tp_name); PyArray_DTypeMeta *dtype_class = PyDataMem_NEW(sizeof(PyArray_DTypeMeta)); if (dtype_class == NULL) { + PyDataMem_FREE(tp_name); return -1; } /* From 63ed08c23163f6fa20a006c2f5a9d9f19227d86e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sun, 9 Feb 2020 12:46:26 -0800 Subject: [PATCH 0005/4003] Create a static prototype to copy, so that it is more obvious what the intention is --- numpy/core/src/multiarray/dtypemeta.c | 47 ++++++++++++++++++--------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index b18ef0c3a08c..73aa79a669e3 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -141,8 +141,9 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) /* * Note: we have no intention of freeing the memory again since this * behaves identically to static type definition (see comment above). - * This is much cleaner for the legacy API, in the new API both static - * and heap types are possible. + * This is seems cleaner for the legacy API, in the new API both static + * and heap types are possible (some difficulty arises from the fact that + * these are instances of DTypeMeta and not type). * In particular our own DTypes can be true static declarations. * However, this function remains necessary for legacy user dtypes. */ @@ -154,35 +155,49 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) snprintf(tp_name, 100, "numpy.dtype[%s]", descr->typeobj->tp_name); - PyArray_DTypeMeta *dtype_class = PyDataMem_NEW(sizeof(PyArray_DTypeMeta)); + PyArray_DTypeMeta *dtype_class = malloc(sizeof(PyArray_DTypeMeta)); if (dtype_class == NULL) { PyDataMem_FREE(tp_name); return -1; } /* - * Initialize the struct fields similar to static code: + * Initialize the struct fields identically to static code by copying + * a prototype instances for everything except our own fields which + * vary between the DTypes. + * In particular any Object initialization must be strictly copied from + * the untouched prototype to avoid complexities (e.g. with PyPy). + * Any Type slots need to be fixed before PyType_Ready, although most + * will be inherited automatically there. */ - /* Copy in np.dtype since it shares most things */ - memcpy(dtype_class, &PyArrayDescr_Type, sizeof(PyArray_DTypeMeta)); - - /* Fix name, base, and __new__*/ + static PyArray_DTypeMeta prototype = { + {{ + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_name = NULL, /* set below */ + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_base = &PyArrayDescr_Type, + .tp_new = (newfunc)legacy_dtype_default_new + },}, + .is_legacy = 1, + .is_abstract = 0, /* this is a concrete DType */ + /* Further fields are not common between DTypes */ + }; + memcpy(dtype_class, &prototype, sizeof(PyArray_DTypeMeta)); + /* Fix name of the Type*/ ((PyTypeObject *)dtype_class)->tp_name = tp_name; - ((PyTypeObject *)dtype_class)->tp_base = &PyArrayDescr_Type; - ((PyTypeObject *)dtype_class)->tp_new = (newfunc)legacy_dtype_default_new; /* Let python finish the initialization (probably unnecessary) */ if (PyType_Ready((PyTypeObject *)dtype_class) < 0) { return -1; } - /* np.dtype is not a concrete DType, this one is */ - dtype_class->is_abstract = NPY_FALSE; - - Py_INCREF(descr); /* descr is a singleton that must survive, ensure. */ + /* + * Fill DTypeMeta information that varies between DTypes, any variable + * type information would need to be set before PyType_Ready(). + */ dtype_class->singleton = descr; Py_INCREF(descr->typeobj); dtype_class->scalar_type = descr->typeobj; - dtype_class->is_legacy = NPY_TRUE; dtype_class->type_num = descr->type_num; dtype_class->type = descr->type; dtype_class->f = descr->f; @@ -193,7 +208,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) /* Datetimes are flexible, but were not considered previously */ dtype_class->is_flexible = NPY_TRUE; } - if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { + else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { dtype_class->is_flexible = NPY_TRUE; dtype_class->itemsize = -1; /* itemsize is not fixed */ } From c3c0786fd24a5302a8bc6762f92a8e78deee1602 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 10 Feb 2020 17:41:32 -0800 Subject: [PATCH 0006/4003] STY: Just small indentation fixup --- numpy/core/src/multiarray/descriptor.c | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index e89c4f6382b7..5bf537816e43 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -3469,23 +3469,24 @@ static PyMappingMethods descr_as_mapping = { * NOTE: Since this is a MetaClass, the name has Full appended here, the * correct name of the type is PyArrayDescr_Type. */ -NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = {{{ - /* The NULL represents `type`, this is set to DTypeMeta at import time */ - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "numpy.dtype", - .tp_basicsize = sizeof(PyArray_Descr), - .tp_dealloc = (destructor)arraydescr_dealloc, - .tp_repr = (reprfunc)arraydescr_repr, - .tp_as_number = &descr_as_number, - .tp_as_sequence = &descr_as_sequence, - .tp_as_mapping = &descr_as_mapping, - .tp_str = (reprfunc)arraydescr_str, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_richcompare = (richcmpfunc)arraydescr_richcompare, - .tp_methods = arraydescr_methods, - .tp_members = arraydescr_members, - .tp_getset = arraydescr_getsets, - .tp_new = arraydescr_new, +NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = { + {{ + /* NULL represents `type`, this is set to DTypeMeta at import time */ + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "numpy.dtype", + .tp_basicsize = sizeof(PyArray_Descr), + .tp_dealloc = (destructor)arraydescr_dealloc, + .tp_repr = (reprfunc)arraydescr_repr, + .tp_as_number = &descr_as_number, + .tp_as_sequence = &descr_as_sequence, + .tp_as_mapping = &descr_as_mapping, + .tp_str = (reprfunc)arraydescr_str, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_richcompare = (richcmpfunc)arraydescr_richcompare, + .tp_methods = arraydescr_methods, + .tp_members = arraydescr_members, + .tp_getset = arraydescr_getsets, + .tp_new = arraydescr_new, },}, .type_num = -1, .kind = '\0', From e02cae81b82b1c45ffeb62875934dfc49b7b779a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 18 Feb 2020 13:53:40 -0800 Subject: [PATCH 0007/4003] Add tests and fixup __name__ --- numpy/core/src/multiarray/dtypemeta.c | 22 ++++++++++++++++++---- numpy/core/tests/test_dtype.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 73aa79a669e3..23c39606f077 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -147,13 +147,27 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) * In particular our own DTypes can be true static declarations. * However, this function remains necessary for legacy user dtypes. */ - char *tp_name = PyDataMem_NEW(100); + + const char *scalar_name = descr->typeobj->tp_name; + /* + * We have to take only the name, and ignore the module to get + * a reasonable __name__, since static types are limited in this regard + * (this is not ideal, but not a big issue in practice). + * This is what Python does to print __name__ for static types. + */ + const char *dot = strrchr(scalar_name, '.'); + if (dot) { + scalar_name = dot + 1; + } + ssize_t name_length = strlen(scalar_name) + 14; + + char *tp_name = malloc(name_length); if (tp_name == NULL) { PyErr_NoMemory(); return -1; } - snprintf(tp_name, 100, "numpy.dtype[%s]", - descr->typeobj->tp_name); + + snprintf(tp_name, name_length, "numpy.dtype[%s]", scalar_name); PyArray_DTypeMeta *dtype_class = malloc(sizeof(PyArray_DTypeMeta)); if (dtype_class == NULL) { @@ -176,7 +190,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) .tp_basicsize = sizeof(PyArray_Descr), .tp_flags = Py_TPFLAGS_DEFAULT, .tp_base = &PyArrayDescr_Type, - .tp_new = (newfunc)legacy_dtype_default_new + .tp_new = (newfunc)legacy_dtype_default_new, },}, .is_legacy = 1, .is_abstract = 0, /* this is a concrete DType */ diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index c9a65cd9ccce..cbe8322a3a8a 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1091,6 +1091,17 @@ class dt(np.void): with pytest.raises(RecursionError): np.dtype(dt(1)) + +class TestDTypeClasses: + @pytest.mark.parametrize("dtype", list(np.typecodes['All']) + [rational]) + def test_dtypes_are_subclasses(self, dtype): + dtype = np.dtype(dtype) + assert isinstance(dtype, np.dtype) + assert type(dtype) is not np.dtype + assert type(dtype).__name__ == f"dtype[{dtype.type.__name__}]" + assert type(dtype).__module__ == "numpy" + + class TestFromCTypes: @staticmethod From 6d951be835308bbabe05694c598f2df96cc11ccf Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 18 Feb 2020 14:07:59 -0800 Subject: [PATCH 0008/4003] Use "parametric" instead of "flexible" and add tests for attributes Note that these attributes are *not* fixed yet, just a start to have something. --- numpy/core/include/numpy/ndarraytypes.h | 8 ++++---- numpy/core/src/multiarray/descriptor.c | 2 +- numpy/core/src/multiarray/dtypemeta.c | 18 +++++++++--------- numpy/core/tests/test_dtype.py | 17 ++++++++++++++++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index db7d7bb30614..63c8963022b6 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1840,9 +1840,9 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, /* * TODO: Update above comment as necessary. - * Most DTypes will have a singleton default instance, for the flexible - * legacy DTypes (bytes, string, void, datetime) this may be a pointer - * to the *prototype* instance? + * Most DTypes will have a singleton default instance, for the + * parametric legacy DTypes (bytes, string, void, datetime) this + * may be a pointer to the *prototype* instance? */ PyArray_Descr *singleton; /* @@ -1865,7 +1865,7 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, char type; /* flags describing data type */ char flags; - npy_bool is_flexible; + npy_bool is_parametric; /* whether the DType can be instantiated (i.e. np.dtype cannot) */ npy_bool is_abstract; /* number representing this type */ diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 5bf537816e43..37373451e3c8 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -3491,5 +3491,5 @@ NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = { .type_num = -1, .kind = '\0', .is_abstract = 1, - .is_flexible = 0, + .is_parametric = 0, }; diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 23c39606f077..943ac299b5eb 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -82,11 +82,11 @@ legacy_dtype_default_new(PyArray_DTypeMeta *self, PyObject *args, PyObject *kwargs) { /* TODO: This should allow endianess and possibly metadata */ - if (self->is_flexible) { - /* reject flexible ones since we would need to get unit, etc. info */ + if (self->is_parametric) { + /* reject parametric ones since we would need to get unit, etc. info */ PyErr_Format(PyExc_TypeError, - "Preliminary-API: Flexible legacy DType '%S' can only be " - "instantiated using `np.dtype(...)`", self); + "Preliminary-API: Flexible/Parametric legacy DType '%S' can " + "only be instantiated using `np.dtype(...)`", self); return NULL; } @@ -220,10 +220,10 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) if (PyTypeNum_ISDATETIME(descr->type_num)) { /* Datetimes are flexible, but were not considered previously */ - dtype_class->is_flexible = NPY_TRUE; + dtype_class->is_parametric = NPY_TRUE; } else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { - dtype_class->is_flexible = NPY_TRUE; + dtype_class->is_parametric = NPY_TRUE; dtype_class->itemsize = -1; /* itemsize is not fixed */ } @@ -240,11 +240,11 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) */ static PyMemberDef dtypemeta_members[] = { {"_abstract", - T_BYTE, offsetof(PyArray_DTypeMeta, is_abstract), READONLY, NULL}, + T_BYTE, offsetof(PyArray_DTypeMeta, is_abstract), READONLY, NULL}, {"type", T_OBJECT, offsetof(PyArray_DTypeMeta, scalar_type), READONLY, NULL}, - {"_flexible", - T_BYTE, offsetof(PyArray_DTypeMeta, is_flexible), READONLY, NULL}, + {"_parametric", + T_BYTE, offsetof(PyArray_DTypeMeta, is_parametric), READONLY, NULL}, {NULL, 0, 0, 0, NULL}, }; diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index cbe8322a3a8a..d983a03f8ac4 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1094,12 +1094,27 @@ class dt(np.void): class TestDTypeClasses: @pytest.mark.parametrize("dtype", list(np.typecodes['All']) + [rational]) - def test_dtypes_are_subclasses(self, dtype): + def test_basic_dtypes_subclass_properties(self, dtype): + # Note: Except for the isinstance and type checks, these attributes + # are considered currently private and may change. dtype = np.dtype(dtype) assert isinstance(dtype, np.dtype) assert type(dtype) is not np.dtype assert type(dtype).__name__ == f"dtype[{dtype.type.__name__}]" assert type(dtype).__module__ == "numpy" + assert not type(dtype)._abstract + + parametric = (np.void, np.str_, np.bytes_, np.datetime64, np.timedelta64) + if dtype.type not in parametric: + assert not type(dtype)._parametric + assert type(dtype)() is dtype + else: + assert type(dtype)._parametric + with assert_raises(TypeError): + type(dtype)() + + def test_dtype_superclass(self): + assert np.dtype._abstract class TestFromCTypes: From 7c365448314fc47f6bc99c562caf238efe986b36 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 18 Feb 2020 14:21:02 -0800 Subject: [PATCH 0009/4003] Improve docs and test slightl; Reject also kwargs in dtype from type creation --- numpy/core/src/multiarray/dtypemeta.c | 7 ++++--- numpy/core/tests/test_dtype.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 943ac299b5eb..7c03e8a46e47 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -67,8 +67,8 @@ dtypemeta_traverse(PyArray_DTypeMeta *type, visitproc visit, void *arg) /* * We have to traverse the base class (if it is a HeapType). * PyType_Type will handle this logic for us. - * TODO: In the future, we may have to VISIT some python objects held, - * however, only if we are a Py_TPFLAGS_HEAPTYPE. + * This function is currently not used, but may be necessary in the future + * when we implement HeapTypes (python/dynamically defined types). */ assert(!type->is_legacy && (PyTypeObject *)type != &PyArrayDescr_Type); Py_VISIT(type->singleton); @@ -90,7 +90,8 @@ legacy_dtype_default_new(PyArray_DTypeMeta *self, return NULL; } - if (PyTuple_GET_SIZE(args) != 0) { + if (PyTuple_GET_SIZE(args) != 0 || + (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { PyErr_Format(PyExc_TypeError, "currently only the no-argument instantiation is supported; " "use `np.dtype` instead."); diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index d983a03f8ac4..73aa01de628c 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1104,6 +1104,9 @@ def test_basic_dtypes_subclass_properties(self, dtype): assert type(dtype).__module__ == "numpy" assert not type(dtype)._abstract + # the flexible dtypes and datetime/timedelta have additional parameters + # which are more than just storage information, these would need to be + # given when creating a dtype: parametric = (np.void, np.str_, np.bytes_, np.datetime64, np.timedelta64) if dtype.type not in parametric: assert not type(dtype)._parametric @@ -1114,6 +1117,11 @@ def test_basic_dtypes_subclass_properties(self, dtype): type(dtype)() def test_dtype_superclass(self): + assert type(np.dtype) is not type + assert isinstance(np.dtype, type) + + assert type(np.dtype).__name__ == "_DTypeMeta" + assert type(np.dtype).__module__ == "numpy" assert np.dtype._abstract From e6fde0eb58fe10a7cc58beee61ca77f269223b66 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 18 Feb 2020 20:43:47 -0800 Subject: [PATCH 0010/4003] PyDict_GET_SIZE is not public (or at least not on older python) --- numpy/core/src/multiarray/dtypemeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 7c03e8a46e47..daaf58aa69c6 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -91,7 +91,7 @@ legacy_dtype_default_new(PyArray_DTypeMeta *self, } if (PyTuple_GET_SIZE(args) != 0 || - (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { + (kwargs != NULL && PyDict_Size(kwargs))) { PyErr_Format(PyExc_TypeError, "currently only the no-argument instantiation is supported; " "use `np.dtype` instead."); From 2ea745b41e093fa4c100a074be9392a1b44d1f6f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 18 Mar 2020 17:29:22 -0500 Subject: [PATCH 0011/4003] small fixups/name changes --- numpy/core/include/numpy/ndarraytypes.h | 23 +++---- numpy/core/src/multiarray/descriptor.c | 6 +- numpy/core/src/multiarray/dtypemeta.c | 79 +++++++++++++------------ 3 files changed, 52 insertions(+), 56 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 63c8963022b6..5662ca7a38dd 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1839,18 +1839,20 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, PyHeapTypeObject super; /* - * TODO: Update above comment as necessary. * Most DTypes will have a singleton default instance, for the * parametric legacy DTypes (bytes, string, void, datetime) this * may be a pointer to the *prototype* instance? */ PyArray_Descr *singleton; /* - * Is this DType created using the old API? - * TODO: For now this mostly exists for possible assertions, - * we may not need it. + * Is this DType created using the old API? This exists mainly to + * allow for assertions in paths specific to wrapping legacy types. */ - npy_bool is_legacy; + npy_bool legacy; + /* The values stored by a parametric datatype depend on its instance */ + npy_bool parametric; + /* whether the DType can be instantiated (i.e. np.dtype cannot) */ + npy_bool abstract; /* * The following fields replicate the most important dtype information. @@ -1865,20 +1867,11 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, char type; /* flags describing data type */ char flags; - npy_bool is_parametric; - /* whether the DType can be instantiated (i.e. np.dtype cannot) */ - npy_bool is_abstract; /* number representing this type */ int type_num; - /* - * itemsize of this type, -1 if variable. Use npy_intp, even though it - * is effectively limited to int due to other public API. - */ - npy_intp itemsize; /* * Point to the original ArrFuncs. - * TODO: If we wanted to do, we could make a copy to detect changes. - * However, I doubt that is necessary. + * NOTE: We could make a copy to detect changes to `f`. */ PyArray_ArrFuncs *f; } PyArray_DTypeMeta; diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 37373451e3c8..b4107f8f334b 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -3490,6 +3490,8 @@ NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = { },}, .type_num = -1, .kind = '\0', - .is_abstract = 1, - .is_parametric = 0, + .abstract = 1, + .parametric = 0, + .singleton = 0, + .scalar_type = NULL, }; diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index daaf58aa69c6..76f7b599a520 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -15,17 +15,17 @@ static void dtypemeta_dealloc(PyArray_DTypeMeta *self) { - /* - * PyType_Type asserts Py_TPFLAGS_HEAPTYPE as well. Do not rely on - * a python debug build though. - */ + /* Do not accidentally delete a statically defined DType: */ assert(((PyTypeObject *)self)->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_XDECREF(self->scalar_type); + Py_XDECREF(self->singleton); PyType_Type.tp_dealloc((PyObject *) self); } static PyObject * -dtypemeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +dtypemeta_new(PyTypeObject *NPY_UNUSED(type), + PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) { PyErr_SetString(PyExc_TypeError, "Preliminary-API: Cannot subclass DType."); @@ -33,10 +33,11 @@ dtypemeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -dtypemeta_init(PyTypeObject *type, PyObject *args, PyObject *kwds) +dtypemeta_init(PyTypeObject *NPY_UNUSED(type), + PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) { PyErr_SetString(PyExc_TypeError, - "Preliminary-API: Cannot initialize DType class."); + "Preliminary-API: Cannot __init__ DType class."); return -1; } @@ -46,8 +47,8 @@ dtypemeta_init(PyTypeObject *type, PyObject *args, PyObject *kwds) * * Python Type objects are either statically created (typical C-Extension type) * or HeapTypes (typically created in Python). - * HeapTypes have the Py_TPFLAGS_HEAPTYPE flag, and are garbage collected, our - * DTypeMeta instances (`np.dtype` and its subclasses) *may* be HeapTypes + * HeapTypes have the Py_TPFLAGS_HEAPTYPE flag and are garbage collected. + * Our DTypeMeta instances (`np.dtype` and its subclasses) *may* be HeapTypes * if the Py_TPFLAGS_HEAPTYPE flag is set (they are created from Python). * They are not for legacy DTypes or np.dtype itself. * @@ -67,10 +68,12 @@ dtypemeta_traverse(PyArray_DTypeMeta *type, visitproc visit, void *arg) /* * We have to traverse the base class (if it is a HeapType). * PyType_Type will handle this logic for us. - * This function is currently not used, but may be necessary in the future - * when we implement HeapTypes (python/dynamically defined types). + * This function is currently not used, but will probably be necessary + * in the future when we implement HeapTypes (python/dynamically + * defined types). It should be revised at that time. */ - assert(!type->is_legacy && (PyTypeObject *)type != &PyArrayDescr_Type); + assert(0); + assert(!type->legacy && (PyTypeObject *)type != &PyArrayDescr_Type); Py_VISIT(type->singleton); Py_VISIT(type->scalar_type); return PyType_Type.tp_traverse((PyObject *)type, visit, arg); @@ -82,7 +85,7 @@ legacy_dtype_default_new(PyArray_DTypeMeta *self, PyObject *args, PyObject *kwargs) { /* TODO: This should allow endianess and possibly metadata */ - if (self->is_parametric) { + if (self->parametric) { /* reject parametric ones since we would need to get unit, etc. info */ PyErr_Format(PyExc_TypeError, "Preliminary-API: Flexible/Parametric legacy DType '%S' can " @@ -193,8 +196,8 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) .tp_base = &PyArrayDescr_Type, .tp_new = (newfunc)legacy_dtype_default_new, },}, - .is_legacy = 1, - .is_abstract = 0, /* this is a concrete DType */ + .legacy = 1, + .abstract = 0, /* this is a concrete DType */ /* Further fields are not common between DTypes */ }; memcpy(dtype_class, &prototype, sizeof(PyArray_DTypeMeta)); @@ -217,15 +220,13 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) dtype_class->type = descr->type; dtype_class->f = descr->f; dtype_class->kind = descr->kind; - dtype_class->itemsize = descr->elsize; if (PyTypeNum_ISDATETIME(descr->type_num)) { /* Datetimes are flexible, but were not considered previously */ - dtype_class->is_parametric = NPY_TRUE; + dtype_class->parametric = NPY_TRUE; } else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { - dtype_class->is_parametric = NPY_TRUE; - dtype_class->itemsize = -1; /* itemsize is not fixed */ + dtype_class->parametric = NPY_TRUE; } /* Finally, replace the current class of the descr */ @@ -240,29 +241,29 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) * preliminary (the flags should also return bools). */ static PyMemberDef dtypemeta_members[] = { - {"_abstract", - T_BYTE, offsetof(PyArray_DTypeMeta, is_abstract), READONLY, NULL}, - {"type", - T_OBJECT, offsetof(PyArray_DTypeMeta, scalar_type), READONLY, NULL}, - {"_parametric", - T_BYTE, offsetof(PyArray_DTypeMeta, is_parametric), READONLY, NULL}, - {NULL, 0, 0, 0, NULL}, + {"_abstract", + T_BYTE, offsetof(PyArray_DTypeMeta, abstract), READONLY, NULL}, + {"type", + T_OBJECT, offsetof(PyArray_DTypeMeta, scalar_type), READONLY, NULL}, + {"_parametric", + T_BYTE, offsetof(PyArray_DTypeMeta, parametric), READONLY, NULL}, + {NULL, 0, 0, 0, NULL}, }; NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "numpy._DTypeMeta", - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_dealloc = (destructor)dtypemeta_dealloc, - /* Types are garbage collected (see dtypemeta_is_gc documentation) */ - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", - .tp_members = dtypemeta_members, - .tp_base = NULL, /* set to PyType_Type at import time */ - .tp_init = (initproc)dtypemeta_init, - .tp_new = dtypemeta_new, - .tp_is_gc = dtypemeta_is_gc, - .tp_traverse = (traverseproc)dtypemeta_traverse, + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "numpy._DTypeMeta", + .tp_basicsize = sizeof(PyArray_DTypeMeta), + .tp_dealloc = (destructor)dtypemeta_dealloc, + /* Types are garbage collected (see dtypemeta_is_gc documentation) */ + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", + .tp_members = dtypemeta_members, + .tp_base = NULL, /* set to PyType_Type at import time */ + .tp_init = (initproc)dtypemeta_init, + .tp_new = dtypemeta_new, + .tp_is_gc = dtypemeta_is_gc, + .tp_traverse = (traverseproc)dtypemeta_traverse, }; From 1511c1f267c35b8e619b3b1c913f336ae7505910 Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Thu, 16 Apr 2020 14:52:14 +0200 Subject: [PATCH 0012/4003] ENH: improve printing of arrays with multi-line reprs --- numpy/core/arrayprint.py | 31 +++++++++++++++++++++++++---- numpy/core/tests/test_arrayprint.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 456ef76f005b..b0800570051c 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -695,7 +695,7 @@ def array2string(a, max_line_width=None, precision=None, def _extendLine(s, line, word, line_width, next_line_prefix, legacy): needs_wrap = len(line) + len(word) > line_width if legacy != '1.13': - s# don't wrap lines if it won't help + # don't wrap lines if it won't help if len(line) <= len(next_line_prefix): needs_wrap = False @@ -706,6 +706,27 @@ def _extendLine(s, line, word, line_width, next_line_prefix, legacy): return s, line +def _extendLine_pretty(s, line, word, line_width, next_line_prefix, legacy): + """ + Extends line with nicely formatted (possibly multi-line) string ``word``. + """ + if legacy == '1.13': + return _extendLine(s, line, word, line_width, next_line_prefix, legacy) + + words = word.splitlines() + line_length = len(line) + s, line = _extendLine( + s, line, words[0], line_width, next_line_prefix, legacy) + for word in words[1::]: + s += line.rstrip() + '\n' + if line_length + len(word) > line_width \ + and line_length > len(next_line_prefix): + line = next_line_prefix + word + else: + line = line_length*' ' + word + + return s, line + def _formatArray(a, format_function, line_width, next_line_prefix, separator, edge_items, summary_insert, legacy): """formatArray is designed for two modes of operation: @@ -758,7 +779,7 @@ def recurser(index, hanging_indent, curr_width): line = hanging_indent for i in range(leading_items): word = recurser(index + (i,), next_hanging_indent, next_width) - s, line = _extendLine( + s, line = _extendLine_pretty( s, line, word, elem_width, hanging_indent, legacy) line += separator @@ -772,7 +793,7 @@ def recurser(index, hanging_indent, curr_width): for i in range(trailing_items, 1, -1): word = recurser(index + (-i,), next_hanging_indent, next_width) - s, line = _extendLine( + s, line = _extendLine_pretty( s, line, word, elem_width, hanging_indent, legacy) line += separator @@ -780,7 +801,7 @@ def recurser(index, hanging_indent, curr_width): # width of the separator is not considered on 1.13 elem_width = curr_width word = recurser(index + (-1,), next_hanging_indent, next_width) - s, line = _extendLine( + s, line = _extendLine_pretty( s, line, word, elem_width, hanging_indent, legacy) s += line @@ -824,6 +845,8 @@ def recurser(index, hanging_indent, curr_width): # performance and PyPy friendliness, we break the cycle: recurser = None + + def _none_or_positive_arg(x, name): if x is None: return -1 diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index e292174612c3..b14fcbf49114 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -395,6 +395,37 @@ def test_wide_element(self): "[ 'xxxxx']" ) + def test_multiline_repr(self): + class MultiLine: + def __repr__(self): + return "Line 1\nLine 2" + + a = np.array([[None, MultiLine()], [MultiLine(), None]]) + + assert_equal( + np.array2string(a), + '[[None Line 1\n' + ' Line 2]\n' + ' [Line 1\n' + ' Line 2 None]]' + ) + assert_equal( + np.array2string(a, max_line_width=5), + '[[None\n' + ' Line 1\n' + ' Line 2]\n' + ' [Line 1\n' + ' Line 2\n' + ' None]]' + ) + assert_equal( + np.array_repr(a), + 'array([[None, Line 1\n' + ' Line 2],\n' + ' [Line 1\n' + ' Line 2, None]], dtype=object)' + ) + @given(hynp.from_dtype(np.dtype("U"))) def test_any_text(self, text): # This test checks that, given any value that can be represented in an From 947b6a6380c08b02047beb4c34466c6d2bdc9ffe Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Thu, 16 Apr 2020 18:05:14 +0200 Subject: [PATCH 0013/4003] Improve testing --- numpy/core/arrayprint.py | 2 -- numpy/core/tests/test_arrayprint.py | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index b0800570051c..01db605f64b7 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -845,8 +845,6 @@ def recurser(index, hanging_indent, curr_width): # performance and PyPy friendliness, we break the cycle: recurser = None - - def _none_or_positive_arg(x, name): if x is None: return -1 diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index b14fcbf49114..64ef5b14ec2a 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -419,13 +419,30 @@ def __repr__(self): ' None]]' ) assert_equal( - np.array_repr(a), + repr(a), 'array([[None, Line 1\n' ' Line 2],\n' ' [Line 1\n' ' Line 2, None]], dtype=object)' ) + def test_nested_array_repr(self): + a = np.empty((2, 2), dtype=object) + a[0, 0] = np.eye(2) + a[0, 1] = np.eye(3) + a[1, 0] = None + a[1, 1] = np.ones((3, 1)) + assert_equal( + repr(a), + 'array([[array([[1., 0.],\n' + ' [0., 1.]]), array([[1., 0., 0.],\n' + ' [0., 1., 0.],\n' + ' [0., 0., 1.]])],\n' + ' [None, array([[1.],\n' + ' [1.],\n' + ' [1.]])]], dtype=object)' + ) + @given(hynp.from_dtype(np.dtype("U"))) def test_any_text(self, text): # This test checks that, given any value that can be represented in an From a95aae3acac02e9c8b17e9cb3e460c26e2eba0ae Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Fri, 17 Apr 2020 09:10:26 +0200 Subject: [PATCH 0014/4003] Improve way of breaking lines --- numpy/core/arrayprint.py | 25 ++++++++++++++++--------- numpy/core/tests/test_arrayprint.py | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 01db605f64b7..0f3e5ea5c7b2 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -710,20 +710,27 @@ def _extendLine_pretty(s, line, word, line_width, next_line_prefix, legacy): """ Extends line with nicely formatted (possibly multi-line) string ``word``. """ - if legacy == '1.13': + words = word.splitlines() + if len(words) == 1 or legacy == '1.13': return _extendLine(s, line, word, line_width, next_line_prefix, legacy) - words = word.splitlines() line_length = len(line) - s, line = _extendLine( - s, line, words[0], line_width, next_line_prefix, legacy) + max_word_length = max(len(word) for word in words) + if line_length + max_word_length > line_width and \ + len(line) > len(next_line_prefix): + s += line.rstrip() + '\n' + line = next_line_prefix + words[0] + indent = next_line_prefix + else: + line += words[0] + indent = line_length*' ' + for word in words[1::]: s += line.rstrip() + '\n' - if line_length + len(word) > line_width \ - and line_length > len(next_line_prefix): - line = next_line_prefix + word - else: - line = line_length*' ' + word + line = indent + word + + suffix_length = max_word_length-len(words[-1]) + line += suffix_length*' ' return s, line diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index 64ef5b14ec2a..a2703d81b2cc 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -426,6 +426,33 @@ def __repr__(self): ' Line 2, None]], dtype=object)' ) + class MultiLineLong: + def __repr__(self): + return "Line 1\nLooooooooooongestLine2\nLongerLine 3" + + a = np.array([[None, MultiLineLong()], [MultiLineLong(), None]]) + assert_equal( + repr(a), + 'array([[None, Line 1\n' + ' LooooooooooongestLine2\n' + ' LongerLine 3 ],\n' + ' [Line 1\n' + ' LooooooooooongestLine2\n' + ' LongerLine 3 , None]], dtype=object)' + ) + assert_equal( + np.array_repr(a, 20), + 'array([[None,\n' + ' Line 1\n' + ' LooooooooooongestLine2\n' + ' LongerLine 3 ],\n' + ' [Line 1\n' + ' LooooooooooongestLine2\n' + ' LongerLine 3 ,\n' + ' None]],\n' + ' dtype=object)' + ) + def test_nested_array_repr(self): a = np.empty((2, 2), dtype=object) a[0, 0] = np.eye(2) From 58b4027ef73866f3e16c91f9ab707b33c9a3fba9 Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Mon, 27 Apr 2020 13:22:14 +0200 Subject: [PATCH 0015/4003] Make requested changes --- numpy/core/arrayprint.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 0f3e5ea5c7b2..5d9642ea81e1 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -714,22 +714,21 @@ def _extendLine_pretty(s, line, word, line_width, next_line_prefix, legacy): if len(words) == 1 or legacy == '1.13': return _extendLine(s, line, word, line_width, next_line_prefix, legacy) - line_length = len(line) max_word_length = max(len(word) for word in words) - if line_length + max_word_length > line_width and \ - len(line) > len(next_line_prefix): + if (len(line) + max_word_length > line_width and + len(line) > len(next_line_prefix)): s += line.rstrip() + '\n' line = next_line_prefix + words[0] indent = next_line_prefix else: + indent = len(line)*' ' line += words[0] - indent = line_length*' ' for word in words[1::]: s += line.rstrip() + '\n' line = indent + word - suffix_length = max_word_length-len(words[-1]) + suffix_length = max_word_length - len(words[-1]) line += suffix_length*' ' return s, line From e95fcba32f123dc3ea6e23f533fedf5aee64815e Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Mon, 4 May 2020 19:04:20 -0400 Subject: [PATCH 0016/4003] DEP: Deprecate `numpy.dual`. Add a deprecation warning in the `numpy.dual` module, and remove the use of `numpy.dual` from the few places where it is used in the numpy code. --- numpy/__init__.py | 4 +++- numpy/dual.py | 8 ++++++++ numpy/lib/function_base.py | 1 - numpy/matrixlib/defmatrix.py | 4 ++-- numpy/random/_generator.pyx | 6 +++--- numpy/random/mtrand.pyx | 2 +- numpy/tests/test_public_api.py | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index 575e8ea3db7a..e6a24f0d1f65 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -79,7 +79,9 @@ show_config Show numpy build configuration dual - Overwrite certain functions with high-performance Scipy tools + Overwrite certain functions with high-performance SciPy tools. + Note: `numpy.dual` is deprecated. Use the functions from NumPy or Scipy + directly instead of importing them from `numpy.dual`. matlib Make everything matrices. __version__ diff --git a/numpy/dual.py b/numpy/dual.py index 92afec52df90..9200a054ec1c 100644 --- a/numpy/dual.py +++ b/numpy/dual.py @@ -10,6 +10,14 @@ .. _Scipy : https://www.scipy.org """ +import warnings + + +warnings.warn('The module numpy.dual is deprecated. Instead of using dual, ' + 'use the functions directly from numpy or scipy.', + category=DeprecationWarning, + stacklevel=2) + # This module should be used for functions both in numpy and scipy if # you want to use the numpy version if available but the scipy version # otherwise. diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 7eeed7825bdd..74ae3ed6e450 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3224,7 +3224,6 @@ def kaiser(M, beta): >>> plt.show() """ - from numpy.dual import i0 if M == 1: return np.array([1.]) n = arange(0, M) diff --git a/numpy/matrixlib/defmatrix.py b/numpy/matrixlib/defmatrix.py index d1a1211aa6f1..ac7d472bc101 100644 --- a/numpy/matrixlib/defmatrix.py +++ b/numpy/matrixlib/defmatrix.py @@ -829,9 +829,9 @@ def I(self): """ M, N = self.shape if M == N: - from numpy.dual import inv as func + from numpy.linalg import inv as func else: - from numpy.dual import pinv as func + from numpy.linalg import pinv as func return asmatrix(func(self)) @property diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 27cb2859e413..274dba8c409f 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -3502,14 +3502,14 @@ cdef class Generator: # GH10839, ensure double to make tol meaningful cov = cov.astype(np.double) if method == 'svd': - from numpy.dual import svd + from numpy.linalg import svd (u, s, vh) = svd(cov) elif method == 'eigh': - from numpy.dual import eigh + from numpy.linalg import eigh # could call linalg.svd(hermitian=True), but that calculates a vh we don't need (s, u) = eigh(cov) else: - from numpy.dual import cholesky + from numpy.linalg import cholesky l = cholesky(cov) # make sure check_valid is ignored whe method == 'cholesky' diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index 0c0c611afcb7..6f2ba871cdd5 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -4049,7 +4049,7 @@ cdef class RandomState: [True, True] # random """ - from numpy.dual import svd + from numpy.linalg import svd # Check preconditions on arguments mean = np.array(mean) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index fb7ec5d83eb4..7ce74bc43ef0 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -154,7 +154,6 @@ def test_NPY_NO_EXPORT(): "doc.structured_arrays", "doc.subclassing", "doc.ufuncs", - "dual", "f2py", "fft", "lib", @@ -258,6 +257,7 @@ def test_NPY_NO_EXPORT(): "distutils.numpy_distribution", "distutils.pathccompiler", "distutils.unixccompiler", + "dual", "f2py.auxfuncs", "f2py.capi_maps", "f2py.cb_rules", From bd58b471ad86ea402a639bbfe517bd61ebb97c31 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Tue, 5 May 2020 10:19:47 -0400 Subject: [PATCH 0017/4003] DOC: dual: Note the deprecation of numpy.dual in the dual module docstring. Also make the correction "Scipy" -> "SciPy" in a few places. --- doc/source/reference/routines.dual.rst | 2 +- numpy/dual.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/doc/source/reference/routines.dual.rst b/doc/source/reference/routines.dual.rst index 4ed7098d6c3a..01814e9a71cb 100644 --- a/doc/source/reference/routines.dual.rst +++ b/doc/source/reference/routines.dual.rst @@ -1,4 +1,4 @@ -Optionally Scipy-accelerated routines (:mod:`numpy.dual`) +Optionally SciPy-accelerated routines (:mod:`numpy.dual`) ********************************************************* .. automodule:: numpy.dual diff --git a/numpy/dual.py b/numpy/dual.py index 9200a054ec1c..eb7e61aac085 100644 --- a/numpy/dual.py +++ b/numpy/dual.py @@ -1,13 +1,19 @@ """ -Aliases for functions which may be accelerated by Scipy. +.. deprecated:: 1.20 -Scipy_ can be built to use accelerated or otherwise improved libraries +*This module is deprecated. Instead of importing functions from* +``numpy.dual``, *the functions should be imported directly from NumPy +or SciPy*. + +Aliases for functions which may be accelerated by SciPy. + +SciPy_ can be built to use accelerated or otherwise improved libraries for FFTs, linear algebra, and special functions. This module allows developers to transparently support these accelerated functions when -scipy is available but still support users who have only installed +SciPy is available but still support users who have only installed NumPy. -.. _Scipy : https://www.scipy.org +.. _SciPy : https://www.scipy.org """ import warnings From 82d4c78818a3d2ee0e7a921cf46ae69edd2b1731 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Tue, 5 May 2020 10:25:28 -0400 Subject: [PATCH 0018/4003] REL: Add a release note about the deprecation of numpy.dual. --- doc/release/upcoming_changes/16156.deprecation.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release/upcoming_changes/16156.deprecation.rst diff --git a/doc/release/upcoming_changes/16156.deprecation.rst b/doc/release/upcoming_changes/16156.deprecation.rst new file mode 100644 index 000000000000..153cc4428486 --- /dev/null +++ b/doc/release/upcoming_changes/16156.deprecation.rst @@ -0,0 +1,5 @@ +Deprecation of `numpy.dual` +--------------------------- +The module `numpy.dual` is deprecated. Instead of importing functions +from `numpy.dual`, the functions should be imported directly from NumPy +or SciPy. From f9ce04212da5d9fedc6aa92494519dca7937bbf8 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 13:51:43 -0700 Subject: [PATCH 0019/4003] WIP: Added blurb about conv. classes to poly pkg docstring. Emphasize that the class-based interface is preferred. --- numpy/polynomial/__init__.py | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/numpy/polynomial/__init__.py b/numpy/polynomial/__init__.py index 4ff2df57ea05..0d0571b2d948 100644 --- a/numpy/polynomial/__init__.py +++ b/numpy/polynomial/__init__.py @@ -12,6 +12,45 @@ implemented as operations on the coefficients. Additional (module-specific) information can be found in the docstring for the module of interest. +This package provides *convenience classes* for each of six different kinds +of polynomials:: + + ============ ================ + Name Provides + ============ ================ + Polynomial Power series + Chebyshev Chebyshev series + Legendre Legendre series + Laguerre Laguerre series + Hermite Hermite series + HermiteE HermiteE series + ============ ================ + +These *convenience classes* provide a consistent interface for creating, +manipulating, and fitting data with polynomials of different bases, and are the +preferred way for interacting with polynomials. The convenience classes are +available from `numpy.polynomial`, eliminating the need to navigate to the +corresponding submodules, e.g. ``np.polynomial.Polynomial`` +or ``np.polynomial.Chebyshev`` instead of +``np.polynomial.polynomial.Polynomial`` or +``np.polynomial.chebyshev.Chebyshev``, respectively. +It is strongly recommended that the class-based interface is used instead of +functions from individual submodules for the sake of consistency and brevity. +For example, to fit a Chebyshev polynomial with degree ``1`` to data given +by arrays ``xdata`` and ``ydata``, the ``fit`` class method:: + + >>> from numpy.polynomial import Chebyshev + >>> c = Chebyshev.fit(xdata, ydata, deg=1) + +is preferred over the ``chebfit`` function from the +`numpy.polynomial.chebyshev` module:: + + >>> from numpy.polynomial.chebyshev import chebfit + >>> c = chebfit(xdata, ydata, deg=1) + +See `routines.polynomials.classes` for a more detailed introduction to the +polynomial convenience classes. + """ from .polynomial import Polynomial from .chebyshev import Chebyshev From ee5f63a9a59228385c48f7fe8def2a63ae3c3789 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 15:32:59 -0700 Subject: [PATCH 0020/4003] Add categorized listing of conv. class attrs/methods. --- numpy/polynomial/__init__.py | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/numpy/polynomial/__init__.py b/numpy/polynomial/__init__.py index 0d0571b2d948..9c84a34b19fd 100644 --- a/numpy/polynomial/__init__.py +++ b/numpy/polynomial/__init__.py @@ -51,6 +51,70 @@ See `routines.polynomials.classes` for a more detailed introduction to the polynomial convenience classes. +Convenience Classes +=================== + +The following listing the various constants and methods common to the all of +the classes representing the various kinds of polynomials. In the following, +the term ``Poly`` represents any one of the convenience classes (e.g. +``Polynomial``, ``Chebyshev``, ``Hermite``, etc.) while the lowercase ``p`` +represents an **instance**. + +Constants +--------- + +- ``Poly.domain`` -- Default domain +- ``Poly.window`` -- Default window +- ``Poly.basis_name`` -- String used to represent the basis +- ``Poly.maxpower`` -- Maximum value ``n`` such that ``p(x)**n`` is allowed + (default = 100) +- ``Poly.nickname`` -- String used in printing + +Creation +-------- + +Methods for creating polynomial instances. + +- ``Poly.basis(degree)`` -- Basis polynomial of given degree +- ``Poly.identity() -- Polynomial ``p`` where ``p(x) = x`` for all + ``x`` +- ``Poly.fit(x, y, deg) -- Polynomial of degree ``deg`` with coefficients + determined by the least-squares fit to data + ``x``, ``y`` +- ``Poly.fromroots(roots)`` -- Polynomial with specified roots +- ``p.copy()`` -- Create a copy of ``p`` + +Conversion +---------- + +Methods for converting a polynomial instance of one kind to another. + +- ``p.cast(Poly)`` -- Convert ``p`` to instance of kind ``Poly`` +- ``p.convert(Poly)`` -- Convert ``p`` to instance of kind ``Poly`` or map + between domain and window + +Calculus +-------- +- ``p.deriv()`` -- Take the derivative of ``p`` +- ``p.integ()`` -- Integrate ``p`` + +Validation +---------- +- ``Poly.has_samecoef(p1, p2)`` -- Check if coefficients match +- ``Poly.has_samedomain(p1, p2)`` -- Check if domains match +- ``Poly.has_sametype(p1, p2)`` -- Check if types match +- ``Poly.has_samewindow(p1, p2)`` -- Check if windows match + +Misc +---- +- ``p.linspace`` -- Return ``x, p(x)`` at equally-spaced points in domain +- ``p.mapparms`` -- Return the parameters for the linear mapping between + ``domain`` and ``window``. +- ``p.roots`` -- Return the roots of `p`. +- ``p.trim`` -- Remove trailing coefficients. +- ``p.cutdeg(degree)`` -- Truncate p to given degree +- ``p.truncate(size)`` -- Truncate p to given size + """ from .polynomial import Polynomial from .chebyshev import Chebyshev From 20701a873a91f5f2f983dce84248e5a7d3d458dd Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 15:33:21 -0700 Subject: [PATCH 0021/4003] Remove unused polynomial package reference doc --- doc/source/reference/routines.polynomials.package.rst | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 doc/source/reference/routines.polynomials.package.rst diff --git a/doc/source/reference/routines.polynomials.package.rst b/doc/source/reference/routines.polynomials.package.rst deleted file mode 100644 index ca1217f80050..000000000000 --- a/doc/source/reference/routines.polynomials.package.rst +++ /dev/null @@ -1,6 +0,0 @@ -:orphan: - -.. automodule:: numpy.polynomial - :no-members: - :no-inherited-members: - :no-special-members: From c2b6402d2e184ce732cba6f9c8e1af24db1d1c60 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 15:38:45 -0700 Subject: [PATCH 0022/4003] Revert "Remove unused polynomial package reference doc" This reverts commit 82392186a2a139f3f5bbf9a5bdff8e3b20c9175f. --- doc/source/reference/routines.polynomials.package.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/source/reference/routines.polynomials.package.rst diff --git a/doc/source/reference/routines.polynomials.package.rst b/doc/source/reference/routines.polynomials.package.rst new file mode 100644 index 000000000000..ca1217f80050 --- /dev/null +++ b/doc/source/reference/routines.polynomials.package.rst @@ -0,0 +1,6 @@ +:orphan: + +.. automodule:: numpy.polynomial + :no-members: + :no-inherited-members: + :no-special-members: From 50ca544916f30f22f9c528b836822e91a3963d23 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 16:11:58 -0700 Subject: [PATCH 0023/4003] Fixup rst formatting. --- numpy/polynomial/__init__.py | 54 +++++++++++++++++------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/numpy/polynomial/__init__.py b/numpy/polynomial/__init__.py index 9c84a34b19fd..1b65a2205d00 100644 --- a/numpy/polynomial/__init__.py +++ b/numpy/polynomial/__init__.py @@ -13,10 +13,10 @@ information can be found in the docstring for the module of interest. This package provides *convenience classes* for each of six different kinds -of polynomials:: +of polynomials: ============ ================ - Name Provides + **Name** **Provides** ============ ================ Polynomial Power series Chebyshev Chebyshev series @@ -27,29 +27,30 @@ ============ ================ These *convenience classes* provide a consistent interface for creating, -manipulating, and fitting data with polynomials of different bases, and are the -preferred way for interacting with polynomials. The convenience classes are -available from `numpy.polynomial`, eliminating the need to navigate to the -corresponding submodules, e.g. ``np.polynomial.Polynomial`` -or ``np.polynomial.Chebyshev`` instead of +manipulating, and fitting data with polynomials of different bases. +The convenience classes are the preferred interface for the `~numpy.polynomial` +package, and are available from the `numpy.polynomial` namespace. +This eliminates the need to +navigate to the corresponding submodules, e.g. ``np.polynomial.Polynomial`` +or ``np.polynomial.Chebyshev`` instead of ``np.polynomial.polynomial.Polynomial`` or ``np.polynomial.chebyshev.Chebyshev``, respectively. -It is strongly recommended that the class-based interface is used instead of -functions from individual submodules for the sake of consistency and brevity. +The classes provide a more consistent and concise interface than the +type-specific functions defined in the submodules for each type of polynomial. For example, to fit a Chebyshev polynomial with degree ``1`` to data given -by arrays ``xdata`` and ``ydata``, the ``fit`` class method:: +by arrays ``xdata`` and ``ydata``, the +`~chebyshev.Chebyshev.fit` class method:: >>> from numpy.polynomial import Chebyshev >>> c = Chebyshev.fit(xdata, ydata, deg=1) -is preferred over the ``chebfit`` function from the +is preferred over the `chebyshev.chebfit` function from the `numpy.polynomial.chebyshev` module:: >>> from numpy.polynomial.chebyshev import chebfit >>> c = chebfit(xdata, ydata, deg=1) -See `routines.polynomials.classes` for a more detailed introduction to the -polynomial convenience classes. +See :doc:`routines.polynomials.classes` for more details. Convenience Classes =================== @@ -58,7 +59,7 @@ the classes representing the various kinds of polynomials. In the following, the term ``Poly`` represents any one of the convenience classes (e.g. ``Polynomial``, ``Chebyshev``, ``Hermite``, etc.) while the lowercase ``p`` -represents an **instance**. +represents an **instance** of a polynomial class. Constants --------- @@ -66,8 +67,7 @@ - ``Poly.domain`` -- Default domain - ``Poly.window`` -- Default window - ``Poly.basis_name`` -- String used to represent the basis -- ``Poly.maxpower`` -- Maximum value ``n`` such that ``p(x)**n`` is allowed - (default = 100) +- ``Poly.maxpower`` -- Maximum value ``n`` such that ``p**n`` is allowed - ``Poly.nickname`` -- String used in printing Creation @@ -76,12 +76,10 @@ Methods for creating polynomial instances. - ``Poly.basis(degree)`` -- Basis polynomial of given degree -- ``Poly.identity() -- Polynomial ``p`` where ``p(x) = x`` for all - ``x`` -- ``Poly.fit(x, y, deg) -- Polynomial of degree ``deg`` with coefficients - determined by the least-squares fit to data - ``x``, ``y`` -- ``Poly.fromroots(roots)`` -- Polynomial with specified roots +- ``Poly.identity()`` -- ``p`` where ``p(x) = x`` for all ``x`` +- ``Poly.fit(x, y, deg)`` -- ``p`` of degree ``deg`` with coefficients + determined by the least-squares fit to the data ``x``, ``y`` +- ``Poly.fromroots(roots)`` -- ``p`` with specified roots - ``p.copy()`` -- Create a copy of ``p`` Conversion @@ -91,7 +89,7 @@ - ``p.cast(Poly)`` -- Convert ``p`` to instance of kind ``Poly`` - ``p.convert(Poly)`` -- Convert ``p`` to instance of kind ``Poly`` or map - between domain and window + between ``domain`` and ``window`` Calculus -------- @@ -107,11 +105,11 @@ Misc ---- -- ``p.linspace`` -- Return ``x, p(x)`` at equally-spaced points in domain -- ``p.mapparms`` -- Return the parameters for the linear mapping between - ``domain`` and ``window``. -- ``p.roots`` -- Return the roots of `p`. -- ``p.trim`` -- Remove trailing coefficients. +- ``p.linspace()`` -- Return ``x, p(x)`` at equally-spaced points in ``domain`` +- ``p.mapparms()`` -- Return the parameters for the linear mapping between + ``domain`` and ``window``. +- ``p.roots()`` -- Return the roots of `p`. +- ``p.trim()`` -- Remove trailing coefficients. - ``p.cutdeg(degree)`` -- Truncate p to given degree - ``p.truncate(size)`` -- Truncate p to given size From 3687b5b5baf499cb7ac718b728386077fa8a3cfa Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 5 May 2020 23:14:02 -0700 Subject: [PATCH 0024/4003] DOC: Add transition note to all lib/poly functions Adds a `.. note::` right after the description to all functions/classes from the numpy.lib.polynomial module notifying the user that the numpy.polynomial package is preferred. --- numpy/lib/polynomial.py | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 5a0fa5431ef3..420ec245b713 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -46,6 +46,12 @@ def poly(seq_of_zeros): """ Find the coefficients of a polynomial with the given sequence of roots. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Returns the coefficients of the polynomial whose leading coefficient is one for the given sequence of zeros (multiple roots must be included in the sequence as many times as their multiplicity; see Examples). @@ -168,6 +174,12 @@ def roots(p): """ Return the roots of a polynomial with coefficients given in p. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + The values in the rank-1 array `p` are coefficients of a polynomial. If the length of `p` is n+1 then the polynomial is described by:: @@ -258,6 +270,12 @@ def polyint(p, m=1, k=None): """ Return an antiderivative (indefinite integral) of a polynomial. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + The returned order `m` antiderivative `P` of polynomial `p` satisfies :math:`\\frac{d^m}{dx^m}P(x) = p(x)` and is defined up to `m - 1` integration constants `k`. The constants determine the low-order @@ -357,6 +375,12 @@ def polyder(p, m=1): """ Return the derivative of the specified order of a polynomial. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Parameters ---------- p : poly1d or sequence @@ -431,6 +455,12 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): """ Least squares polynomial fit. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Fit a polynomial ``p(x) = p[0] * x**deg + ... + p[deg]`` of degree `deg` to points `(x, y)`. Returns a vector of coefficients `p` that minimises the squared error in the order `deg`, `deg-1`, ... `0`. @@ -667,6 +697,12 @@ def polyval(p, x): """ Evaluate a polynomial at specific values. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + If `p` is of length N, this function returns the value: ``p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]`` @@ -744,6 +780,12 @@ def polyadd(a1, a2): """ Find the sum of two polynomials. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Returns the polynomial resulting from the sum of two input polynomials. Each input must be either a poly1d object or a 1D sequence of polynomial coefficients, from highest to lowest degree. @@ -806,6 +848,12 @@ def polysub(a1, a2): """ Difference (subtraction) of two polynomials. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Given two polynomials `a1` and `a2`, returns ``a1 - a2``. `a1` and `a2` can be either array_like sequences of the polynomials' coefficients (including coefficients equal to zero), or `poly1d` objects. @@ -854,6 +902,12 @@ def polymul(a1, a2): """ Find the product of two polynomials. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + Finds the polynomial resulting from the multiplication of the two input polynomials. Each input must be either a poly1d object or a 1D sequence of polynomial coefficients, from highest to lowest degree. @@ -915,6 +969,12 @@ def polydiv(u, v): """ Returns the quotient and remainder of polynomial division. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + The input arrays are the coefficients (including any coefficients equal to zero) of the "numerator" (dividend) and "denominator" (divisor) polynomials, respectively. @@ -1009,6 +1069,12 @@ class poly1d: """ A one-dimensional polynomial class. + .. note:: + This forms part of the old polynomial API. Since version 1.4, the + new polynomial API defined in `numpy.polynomial` is preferred. + A summary of the differences can be found in the + :doc:`transition guide `. + A convenience class, used to encapsulate "natural" operations on polynomials so that said operations may take on their customary form in code (see Examples). From 2190134b8c149ccdd9e6b73d23cfd732d230b06a Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Thu, 7 May 2020 11:08:19 +0200 Subject: [PATCH 0025/4003] Add release note snippet --- doc/source/release/1.19.0-notes.rst | 1 - doc/source/release/1.20.0-notes.rst | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 doc/source/release/1.20.0-notes.rst diff --git a/doc/source/release/1.19.0-notes.rst b/doc/source/release/1.19.0-notes.rst index 6e7fd69d4167..8b4fb921f2fa 100644 --- a/doc/source/release/1.19.0-notes.rst +++ b/doc/source/release/1.19.0-notes.rst @@ -3,4 +3,3 @@ ========================== NumPy 1.19.0 Release Notes ========================== - diff --git a/doc/source/release/1.20.0-notes.rst b/doc/source/release/1.20.0-notes.rst new file mode 100644 index 000000000000..e6cc2118ba7d --- /dev/null +++ b/doc/source/release/1.20.0-notes.rst @@ -0,0 +1,12 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.20.0 Release Notes +========================== + +Improvements +============ + +Improve `array_repr` +---------------------------------------------- +Now `array_repr` returns nicely formatted repr for arrays with objects with multi-line reprs. From 9c83b13ce1b08aed8181d284566d002086393a89 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 27 Feb 2020 16:30:54 -0800 Subject: [PATCH 0026/4003] ENH: Improved __str__, __format__ for polynomials Changes the printing style of instances of the convenience classes in the polynomial package to a more "human-readable" format. __str__ has been modified and __format__ added to ABCPolyBase, modifying the string representation of polynomial instances, e.g. when printed. __repr__ and the _repr_latex method (which is used in the Jupyter environment are unchanged. Two print formats have been added: 'unicode' and 'ascii'. 'unicode' is the default mode on *nix systems, and uses unicode values for numeric subscripts and superscripts in the polynomial expression. The 'ascii' format is the default on Windows (due to font considerations) and uses Python-style syntax to represent powers, e.g. x**2. The default printing style can be controlled at the package-level with the set_default_printstyle function. The ABCPolyBase.__str__ has also been made to respect the linewidth printoption. Other parameters from the printoptions dictionary are not used. Co-Authored-By: Warren Weckesser Co-authored-by: Eric Wieser --- .../upcoming_changes/15666.improvement.rst | 8 + numpy/polynomial/__init__.py | 50 ++++ numpy/polynomial/_polybase.py | 110 ++++++- numpy/polynomial/polynomial.py | 8 + numpy/polynomial/tests/test_printing.py | 271 ++++++++++++++++-- 5 files changed, 424 insertions(+), 23 deletions(-) create mode 100644 doc/release/upcoming_changes/15666.improvement.rst diff --git a/doc/release/upcoming_changes/15666.improvement.rst b/doc/release/upcoming_changes/15666.improvement.rst new file mode 100644 index 000000000000..c42d709527a6 --- /dev/null +++ b/doc/release/upcoming_changes/15666.improvement.rst @@ -0,0 +1,8 @@ +Improved string representation for polynomials (__str__) +-------------------------------------------------------- + +The string representation (``__str__``) of all six polynomial types in +`numpy.polynomial` has been updated to give the polynomial as a mathematical +expression instead of an array of coefficients. Two package-wide formats for +the polynomial expressions are available - one using Unicode characters for +superscripts and subscripts, and another using only ASCII characters. diff --git a/numpy/polynomial/__init__.py b/numpy/polynomial/__init__.py index 4ff2df57ea05..43b2caba396f 100644 --- a/numpy/polynomial/__init__.py +++ b/numpy/polynomial/__init__.py @@ -20,6 +20,56 @@ from .hermite_e import HermiteE from .laguerre import Laguerre + +def set_default_printstyle(style): + """ + Set the default format for the string representation of polynomials. + + Values for ``style`` must be valid inputs to ``__format__``, i.e. 'ascii' + or 'unicode'. + + Parameters + ---------- + style : str + Format string for default printing style. Must be either 'ascii' or + 'unicode'. + + Notes + ----- + The default format depends on the platform: 'unicode' is used on + Unix-based systems and 'ascii' on Windows. This determination is based on + default font support for the unicode superscript and subscript ranges. + + Examples + -------- + >>> p = np.polynomial.Polynomial([1, 2, 3]) + >>> c = np.polynomial.Chebyshev([1, 2, 3]) + >>> np.polynomial.set_default_printstyle('unicode') + >>> print(p) + 1.0 + 2.0·x¹ + 3.0·x² + >>> print(c) + 1.0 + 2.0·T₁(x) + 3.0·T₂(x) + >>> np.polynomial.set_default_printstyle('ascii') + >>> print(p) + 1.0 + 2.0 x**1 + 3.0 x**2 + >>> print(c) + 1.0 + 2.0 T_1(x) + 3.0 T_2(x) + >>> # Formatting supercedes all class/package-level defaults + >>> print(f"{p:unicode}") + 1.0 + 2.0·x¹ + 3.0·x² + """ + if style not in ('unicode', 'ascii'): + raise ValueError( + f"Unsupported format string '{style}'. Valid options are 'ascii' " + f"and 'unicode'" + ) + _use_unicode = True + if style == 'ascii': + _use_unicode = False + from ._polybase import ABCPolyBase + ABCPolyBase._use_unicode = _use_unicode + + from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index 53efbb90f2bc..253aa74e81fc 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -6,6 +6,7 @@ abc module from the stdlib, hence it is only available for Python >= 2.6. """ +import os import abc import numbers @@ -67,6 +68,37 @@ class ABCPolyBase(abc.ABC): # Limit runaway size. T_n^m has degree n*m maxpower = 100 + # Unicode character mappings for improved __str__ + _superscript_mapping = str.maketrans({ + "0": "⁰", + "1": "¹", + "2": "²", + "3": "³", + "4": "⁴", + "5": "⁵", + "6": "⁶", + "7": "⁷", + "8": "⁸", + "9": "⁹" + }) + _subscript_mapping = str.maketrans({ + "0": "₀", + "1": "₁", + "2": "₂", + "3": "₃", + "4": "₄", + "5": "₅", + "6": "₆", + "7": "₇", + "8": "₈", + "9": "₉" + }) + # Some fonts don't support full unicode character ranges necessary for + # the full set of superscripts and subscripts, including common/default + # fonts in Windows shells/terminals. Therefore, default to ascii-only + # printing on windows. + _use_unicode = not os.name == 'nt' + @property @abc.abstractmethod def domain(self): @@ -283,10 +315,82 @@ def __repr__(self): name = self.__class__.__name__ return f"{name}({coef}, domain={domain}, window={window})" + def __format__(self, fmt_str): + if fmt_str == '': + return self.__str__() + if fmt_str not in ('ascii', 'unicode'): + raise ValueError( + f"Unsupported format string '{fmt_str}' passed to " + f"{self.__class__}.__format__. Valid options are " + f"'ascii' and 'unicode'" + ) + if fmt_str == 'ascii': + return self._generate_string(self._str_term_ascii) + return self._generate_string(self._str_term_unicode) + def __str__(self): - coef = str(self.coef) - name = self.nickname - return f"{name}({coef})" + if self._use_unicode: + return self._generate_string(self._str_term_unicode) + return self._generate_string(self._str_term_ascii) + + def _generate_string(self, term_method): + """ + Generate the full string representation of the polynomial, using + ``term_method`` to generate each polynomial term. + """ + # Get configuration for line breaks + linewidth = np.get_printoptions().get('linewidth', 75) + if linewidth < 1: + linewidth = 1 + out = f"{self.coef[0]}" + for i, coef in enumerate(self.coef[1:]): + out += " " + power = str(i + 1) + # Polynomial coefficient + if coef >= 0: + next_term = f"+ {coef}" + else: + next_term = f"- {-coef}" + # Polynomial term + next_term += term_method(power, "x") + # Length of the current line with next term added + line_len = len(out.split('\n')[-1]) + len(next_term) + # If not the last term in the polynomial, it will be two + # characters longer due to the +/- with the next term + if i < len(self.coef[1:]) - 1: + line_len += 2 + # Handle linebreaking + if line_len >= linewidth: + next_term = next_term.replace(" ", "\n", 1) + out += next_term + return out + + @classmethod + def _str_term_unicode(cls, i, arg_str): + """ + String representation of single polynomial term using unicode + characters for superscripts and subscripts. + """ + if cls.basis_name is None: + raise NotImplementedError( + "Subclasses must define either a basis_name, or override " + "_str_term_unicode(cls, i, arg_str)" + ) + return (f"·{cls.basis_name}{i.translate(cls._subscript_mapping)}" + f"({arg_str})") + + @classmethod + def _str_term_ascii(cls, i, arg_str): + """ + String representation of a single polynomial term using ** and _ to + represent superscripts and subscripts, respectively. + """ + if cls.basis_name is None: + raise NotImplementedError( + "Subclasses must define either a basis_name, or override " + "_str_term_ascii(cls, i, arg_str)" + ) + return f" {cls.basis_name}_{i}({arg_str})" @classmethod def _repr_latex_term(cls, i, arg_str, needs_parens): diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 2fb032db36c2..97f1e7dc0fe0 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -1495,6 +1495,14 @@ class Polynomial(ABCPolyBase): window = np.array(polydomain) basis_name = None + @classmethod + def _str_term_unicode(cls, i, arg_str): + return f"·{arg_str}{i.translate(cls._superscript_mapping)}" + + @staticmethod + def _str_term_ascii(i, arg_str): + return f" {arg_str}**{i}" + @staticmethod def _repr_latex_term(i, arg_str, needs_parens): if needs_parens: diff --git a/numpy/polynomial/tests/test_printing.py b/numpy/polynomial/tests/test_printing.py index 049d3af2f152..acccb23f5fae 100644 --- a/numpy/polynomial/tests/test_printing.py +++ b/numpy/polynomial/tests/test_printing.py @@ -1,38 +1,269 @@ +import pytest +from numpy.core import arange, printoptions import numpy.polynomial as poly -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_ -class TestStr: - def test_polynomial_str(self): - res = str(poly.Polynomial([0, 1])) - tgt = 'poly([0. 1.])' +class TestStrUnicodeSuperSubscripts: + + @pytest.fixture(scope='class', autouse=True) + def use_unicode(self): + poly.set_default_printstyle('unicode') + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·x¹ + 3.0·x²"), + ([-1, 0, 3, -1], "-1.0 + 0.0·x¹ + 3.0·x² - 1.0·x³"), + (arange(12), ("0.0 + 1.0·x¹ + 2.0·x² + 3.0·x³ + 4.0·x⁴ + 5.0·x⁵ + " + "6.0·x⁶ + 7.0·x⁷ +\n8.0·x⁸ + 9.0·x⁹ + 10.0·x¹⁰ + " + "11.0·x¹¹")), + )) + def test_polynomial_str(self, inp, tgt): + res = str(poly.Polynomial(inp)) assert_equal(res, tgt) - def test_chebyshev_str(self): - res = str(poly.Chebyshev([0, 1])) - tgt = 'cheb([0. 1.])' + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·T₁(x) + 3.0·T₂(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0·T₁(x) + 3.0·T₂(x) - 1.0·T₃(x)"), + (arange(12), ("0.0 + 1.0·T₁(x) + 2.0·T₂(x) + 3.0·T₃(x) + 4.0·T₄(x) + " + "5.0·T₅(x) +\n6.0·T₆(x) + 7.0·T₇(x) + 8.0·T₈(x) + " + "9.0·T₉(x) + 10.0·T₁₀(x) + 11.0·T₁₁(x)")), + )) + def test_chebyshev_str(self, inp, tgt): + res = str(poly.Chebyshev(inp)) + assert_equal(res, tgt) + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·P₁(x) + 3.0·P₂(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0·P₁(x) + 3.0·P₂(x) - 1.0·P₃(x)"), + (arange(12), ("0.0 + 1.0·P₁(x) + 2.0·P₂(x) + 3.0·P₃(x) + 4.0·P₄(x) + " + "5.0·P₅(x) +\n6.0·P₆(x) + 7.0·P₇(x) + 8.0·P₈(x) + " + "9.0·P₉(x) + 10.0·P₁₀(x) + 11.0·P₁₁(x)")), + )) + def test_legendre_str(self, inp, tgt): + res = str(poly.Legendre(inp)) + assert_equal(res, tgt) + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·H₁(x) + 3.0·H₂(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0·H₁(x) + 3.0·H₂(x) - 1.0·H₃(x)"), + (arange(12), ("0.0 + 1.0·H₁(x) + 2.0·H₂(x) + 3.0·H₃(x) + 4.0·H₄(x) + " + "5.0·H₅(x) +\n6.0·H₆(x) + 7.0·H₇(x) + 8.0·H₈(x) + " + "9.0·H₉(x) + 10.0·H₁₀(x) + 11.0·H₁₁(x)")), + )) + def test_hermite_str(self, inp, tgt): + res = str(poly.Hermite(inp)) + assert_equal(res, tgt) + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·He₁(x) + 3.0·He₂(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0·He₁(x) + 3.0·He₂(x) - 1.0·He₃(x)"), + (arange(12), ("0.0 + 1.0·He₁(x) + 2.0·He₂(x) + 3.0·He₃(x) + " + "4.0·He₄(x) + 5.0·He₅(x) +\n6.0·He₆(x) + 7.0·He₇(x) + " + "8.0·He₈(x) + 9.0·He₉(x) + 10.0·He₁₀(x) +\n" + "11.0·He₁₁(x)")), + )) + def test_hermiteE_str(self, inp, tgt): + res = str(poly.HermiteE(inp)) assert_equal(res, tgt) - def test_legendre_str(self): - res = str(poly.Legendre([0, 1])) - tgt = 'leg([0. 1.])' + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0·L₁(x) + 3.0·L₂(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0·L₁(x) + 3.0·L₂(x) - 1.0·L₃(x)"), + (arange(12), ("0.0 + 1.0·L₁(x) + 2.0·L₂(x) + 3.0·L₃(x) + 4.0·L₄(x) + " + "5.0·L₅(x) +\n6.0·L₆(x) + 7.0·L₇(x) + 8.0·L₈(x) + " + "9.0·L₉(x) + 10.0·L₁₀(x) + 11.0·L₁₁(x)")), + )) + def test_laguerre_str(self, inp, tgt): + res = str(poly.Laguerre(inp)) assert_equal(res, tgt) - def test_hermite_str(self): - res = str(poly.Hermite([0, 1])) - tgt = 'herm([0. 1.])' + +class TestStrAscii: + + @pytest.fixture(scope='class', autouse=True) + def use_unicode(self): + poly.set_default_printstyle('ascii') + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 x**1 + 3.0 x**2"), + ([-1, 0, 3, -1], "-1.0 + 0.0 x**1 + 3.0 x**2 - 1.0 x**3"), + (arange(12), ("0.0 + 1.0 x**1 + 2.0 x**2 + 3.0 x**3 + 4.0 x**4 + " + "5.0 x**5 + 6.0 x**6 +\n7.0 x**7 + 8.0 x**8 + " + "9.0 x**9 + 10.0 x**10 + 11.0 x**11")), + )) + def test_polynomial_str(self, inp, tgt): + res = str(poly.Polynomial(inp)) assert_equal(res, tgt) - def test_hermiteE_str(self): - res = str(poly.HermiteE([0, 1])) - tgt = 'herme([0. 1.])' + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 T_1(x) + 3.0 T_2(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0 T_1(x) + 3.0 T_2(x) - 1.0 T_3(x)"), + (arange(12), ("0.0 + 1.0 T_1(x) + 2.0 T_2(x) + 3.0 T_3(x) + " + "4.0 T_4(x) + 5.0 T_5(x) +\n6.0 T_6(x) + 7.0 T_7(x) + " + "8.0 T_8(x) + 9.0 T_9(x) + 10.0 T_10(x) +\n" + "11.0 T_11(x)")), + )) + def test_chebyshev_str(self, inp, tgt): + res = str(poly.Chebyshev(inp)) assert_equal(res, tgt) - def test_laguerre_str(self): - res = str(poly.Laguerre([0, 1])) - tgt = 'lag([0. 1.])' + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 P_1(x) + 3.0 P_2(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0 P_1(x) + 3.0 P_2(x) - 1.0 P_3(x)"), + (arange(12), ("0.0 + 1.0 P_1(x) + 2.0 P_2(x) + 3.0 P_3(x) + " + "4.0 P_4(x) + 5.0 P_5(x) +\n6.0 P_6(x) + 7.0 P_7(x) + " + "8.0 P_8(x) + 9.0 P_9(x) + 10.0 P_10(x) +\n" + "11.0 P_11(x)")), + )) + def test_legendre_str(self, inp, tgt): + res = str(poly.Legendre(inp)) assert_equal(res, tgt) + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 H_1(x) + 3.0 H_2(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0 H_1(x) + 3.0 H_2(x) - 1.0 H_3(x)"), + (arange(12), ("0.0 + 1.0 H_1(x) + 2.0 H_2(x) + 3.0 H_3(x) + " + "4.0 H_4(x) + 5.0 H_5(x) +\n6.0 H_6(x) + 7.0 H_7(x) + " + "8.0 H_8(x) + 9.0 H_9(x) + 10.0 H_10(x) +\n" + "11.0 H_11(x)")), + )) + def test_hermite_str(self, inp, tgt): + res = str(poly.Hermite(inp)) + assert_equal(res, tgt) + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 He_1(x) + 3.0 He_2(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0 He_1(x) + 3.0 He_2(x) - 1.0 He_3(x)"), + (arange(12), ("0.0 + 1.0 He_1(x) + 2.0 He_2(x) + 3.0 He_3(x) + " + "4.0 He_4(x) +\n5.0 He_5(x) + 6.0 He_6(x) + " + "7.0 He_7(x) + 8.0 He_8(x) + 9.0 He_9(x) +\n" + "10.0 He_10(x) + 11.0 He_11(x)")), + )) + def test_hermiteE_str(self, inp, tgt): + res = str(poly.HermiteE(inp)) + assert_equal(res, tgt) + + @pytest.mark.parametrize(('inp', 'tgt'), ( + ([1, 2, 3], "1.0 + 2.0 L_1(x) + 3.0 L_2(x)"), + ([-1, 0, 3, -1], "-1.0 + 0.0 L_1(x) + 3.0 L_2(x) - 1.0 L_3(x)"), + (arange(12), ("0.0 + 1.0 L_1(x) + 2.0 L_2(x) + 3.0 L_3(x) + " + "4.0 L_4(x) + 5.0 L_5(x) +\n6.0 L_6(x) + 7.0 L_7(x) + " + "8.0 L_8(x) + 9.0 L_9(x) + 10.0 L_10(x) +\n" + "11.0 L_11(x)")), + )) + def test_laguerre_str(self, inp, tgt): + res = str(poly.Laguerre(inp)) + assert_equal(res, tgt) + + +class TestLinebreaking: + poly.set_default_printstyle('ascii') + + def test_single_line_one_less(self): + # With 'ascii' style, len(str(p)) is default linewidth - 1 (i.e. 74) + p = poly.Polynomial([123456789, 123456789, 123456789, 1234, 1]) + assert_equal(len(str(p)), 74) + assert_equal(str(p), ( + '123456789.0 + 123456789.0 x**1 + 123456789.0 x**2 + ' + '1234.0 x**3 + 1.0 x**4' + )) + + def test_num_chars_is_linewidth(self): + # len(str(p)) == default linewidth == 75 + p = poly.Polynomial([123456789, 123456789, 123456789, 1234, 10]) + assert_equal(len(str(p)), 75) + assert_equal(str(p), ( + '123456789.0 + 123456789.0 x**1 + 123456789.0 x**2 + ' + '1234.0 x**3 +\n10.0 x**4' + )) + + def test_first_linebreak_multiline_one_less_than_linewidth(self): + # Multiline str where len(first_line) + len(next_term) == lw - 1 == 74 + p = poly.Polynomial( + [123456789, 123456789, 123456789, 12, 1, 123456789] + ) + assert_equal(len(str(p).split('\n')[0]), 74) + assert_equal(str(p), ( + '123456789.0 + 123456789.0 x**1 + 123456789.0 x**2 + ' + '12.0 x**3 + 1.0 x**4 +\n123456789.0 x**5' + )) + + def test_first_linebreak_multiline_on_linewidth(self): + # First line is one character longer than previous test + p = poly.Polynomial( + [123456789, 123456789, 123456789, 123, 1, 123456789] + ) + assert_equal(str(p), ( + '123456789.0 + 123456789.0 x**1 + 123456789.0 x**2 + ' + '123.0 x**3 +\n1.0 x**4 + 123456789.0 x**5' + )) + + @pytest.mark.parametrize(('lw', 'tgt'), ( + (75, ('0.0 + 10.0 x**1 + 200.0 x**2 + 3000.0 x**3 + 40000.0 x**4 +\n' + '500000.0 x**5 + 600000.0 x**6 + 70000.0 x**7 + 8000.0 x**8 + ' + '900.0 x**9')), + (45, ('0.0 + 10.0 x**1 + 200.0 x**2 + 3000.0 x**3 +\n40000.0 x**4 + ' + '500000.0 x**5 +\n600000.0 x**6 + 70000.0 x**7 + 8000.0 x**8 +\n' + '900.0 x**9')), + (132, ('0.0 + 10.0 x**1 + 200.0 x**2 + 3000.0 x**3 + 40000.0 x**4 + ' + '500000.0 x**5 + 600000.0 x**6 + 70000.0 x**7 + 8000.0 x**8 + ' + '900.0 x**9')), + )) + def test_linewidth_printoption(self, lw, tgt): + p = poly.Polynomial( + [0, 10, 200, 3000, 40000, 500000, 600000, 70000, 8000, 900] + ) + with printoptions(linewidth=lw): + assert_equal(str(p), tgt) + for line in str(p).split('\n'): + assert_(len(line) < lw) + + +def test_set_default_printoptions(): + p = poly.Polynomial([1, 2, 3]) + c = poly.Chebyshev([1, 2, 3]) + poly.set_default_printstyle('ascii') + assert_equal(str(p), "1.0 + 2.0 x**1 + 3.0 x**2") + assert_equal(str(c), "1.0 + 2.0 T_1(x) + 3.0 T_2(x)") + poly.set_default_printstyle('unicode') + assert_equal(str(p), "1.0 + 2.0·x¹ + 3.0·x²") + assert_equal(str(c), "1.0 + 2.0·T₁(x) + 3.0·T₂(x)") + with pytest.raises(ValueError): + poly.set_default_printstyle('invalid_input') + + +def test_complex_coefficients(): + p = poly.Polynomial([0+1j, 1+1j, -2+2j, 3+0j]) + poly.set_default_printstyle('unicode') + assert_equal(str(p), "1j + (1+1j)·x¹ - (2-2j)·x² + (3+0j)·x³") + poly.set_default_printstyle('ascii') + assert_equal(str(p), "1j + (1+1j) x**1 - (2-2j) x**2 + (3+0j) x**3") + + +class TestFormat: + def test_format_unicode(self): + poly.Polynomial._use_unicode = False + p = poly.Polynomial([1, 2, 0, -1]) + assert_equal(format(p, 'unicode'), "1.0 + 2.0·x¹ + 0.0·x² - 1.0·x³") + + def test_format_ascii(self): + poly.Polynomial._use_unicode = True + p = poly.Polynomial([1, 2, 0, -1]) + assert_equal( + format(p, 'ascii'), "1.0 + 2.0 x**1 + 0.0 x**2 - 1.0 x**3" + ) + + def test_empty_formatstr(self): + poly.Polynomial._use_unicode = False + p = poly.Polynomial([1, 2, 3]) + assert_equal(format(p), "1.0 + 2.0 x**1 + 3.0 x**2") + assert_equal(f"{p}", "1.0 + 2.0 x**1 + 3.0 x**2") + + def test_bad_formatstr(self): + p = poly.Polynomial([1, 2, 0, -1]) + with pytest.raises(ValueError): + format(p, '.2f') + class TestRepr: def test_polynomial_str(self): From 9fa7fee0ad8d23bd36ef51b0ec02cf2811f1eec6 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" Date: Mon, 11 May 2020 14:11:59 +0200 Subject: [PATCH 0027/4003] ENH: add full_output to f2py.compile fixes #12756 by providing a straightforward way to return the stdout/stderr from compiling the FORTRAN module to the caller --- numpy/f2py/__init__.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 949bac0ffb66..007ed5d89354 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -21,7 +21,8 @@ def compile(source, extra_args='', verbose=True, source_fn=None, - extension='.f' + extension='.f', + full_output=False ): """ Build extension module from a Fortran 77 source string with f2py. @@ -55,10 +56,19 @@ def compile(source, .. versionadded:: 1.11.0 + full_output : bool, optional + If True, return a `subprocess.CompletedProcess` containing + the stdout and stderr of the compile process, instead of just + the status code. + + .. versionadded:: 1.20.0 + + Returns ------- - result : int - 0 on success + result : int or `subprocess.CompletedProcess` + 0 on success, or a `subprocess.CompletedProcess` if + ``full_output=True`` Examples -------- @@ -95,23 +105,21 @@ def compile(source, '-c', 'import numpy.f2py as f2py2e;f2py2e.main()'] + args try: - output = subprocess.check_output(c) - except subprocess.CalledProcessError as exc: - status = exc.returncode - output = '' + cp = subprocess.run(c, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) except OSError: # preserve historic status code used by exec_command() - status = 127 - output = '' - else: - status = 0 - output = output.decode() + cp = subprocess.CompletedProcess(c, 127, stdout='', stderr='') if verbose: - print(output) + print(cp.stdout.decode()) finally: if source_fn is None: os.remove(fname) - return status + + if full_output: + return cp + else: + return cp.returncode from numpy._pytesttester import PytestTester test = PytestTester(__name__) From d3bfaf80c4feb0254f9089e757734e8feea05425 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 13:37:19 -0500 Subject: [PATCH 0028/4003] MAINT: Simplify undoing of iter-axis-perm in nditer --- numpy/core/src/multiarray/nditer_api.c | 16 +++--- numpy/core/src/multiarray/nditer_constr.c | 61 +++++++---------------- numpy/core/src/multiarray/nditer_impl.h | 34 +++++++++++++ 3 files changed, 57 insertions(+), 54 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index e7fe0fa5080c..a5b5e5c51eff 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -935,13 +935,8 @@ NpyIter_GetShape(NpyIter *iter, npy_intp *outshape) if (itflags&NPY_ITFLAG_HASMULTIINDEX) { perm = NIT_PERM(iter); for(idim = 0; idim < ndim; ++idim) { - npy_int8 p = perm[idim]; - if (p < 0) { - outshape[ndim+p] = NAD_SHAPE(axisdata); - } - else { - outshape[ndim-p-1] = NAD_SHAPE(axisdata); - } + int axis = npyiter_undo_iter_axis_perm(idim, ndim, perm, NULL); + outshape[axis] = NAD_SHAPE(axisdata); NIT_ADVANCE_AXISDATA(axisdata, 1); } @@ -1005,8 +1000,9 @@ NpyIter_CreateCompatibleStrides(NpyIter *iter, perm = NIT_PERM(iter); for(idim = 0; idim < ndim; ++idim) { - npy_int8 p = perm[idim]; - if (p < 0) { + npy_bool flipped; + npy_int8 axis = npyiter_undo_iter_axis_perm(idim, ndim, perm, &flipped); + if (flipped) { PyErr_SetString(PyExc_RuntimeError, "Iterator CreateCompatibleStrides may only be called " "if DONT_NEGATE_STRIDES was used to prevent reverse " @@ -1014,7 +1010,7 @@ NpyIter_CreateCompatibleStrides(NpyIter *iter, return NPY_FAIL; } else { - outstrides[ndim-p-1] = itemsize; + outstrides[axis] = itemsize; } itemsize *= NAD_SHAPE(axisdata); diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index e40a2d5946ec..4701e74d42a3 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1470,8 +1470,9 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf return 0; } for (idim = 0; idim < ondim; ++idim) { - npy_intp bshape = broadcast_shape[idim+ndim-ondim], - op_shape = shape[idim]; + npy_intp bshape = broadcast_shape[idim+ndim-ondim]; + npy_intp op_shape = shape[idim]; + if (bshape == 1) { broadcast_shape[idim+ndim-ondim] = op_shape; } @@ -1486,8 +1487,9 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf int i = axes[idim]; if (i >= 0) { if (i < ondim) { - npy_intp bshape = broadcast_shape[idim], - op_shape = shape[i]; + npy_intp bshape = broadcast_shape[idim]; + npy_intp op_shape = shape[i]; + if (bshape == 1) { broadcast_shape[idim] = op_shape; } @@ -1945,25 +1947,18 @@ npyiter_replace_axisdata(NpyIter *iter, int iop, if (op_axes != NULL) { for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { - npy_int8 p; int i; + npy_bool axis_flipped; npy_intp shape; - /* Apply the perm to get the original axis */ - p = perm[idim]; - if (p < 0) { - i = op_axes[ndim+p]; - } - else { - i = op_axes[ndim-p-1]; - } + /* Apply perm to get the original axis, and check if its flipped */ + i = npyiter_undo_iter_axis_perm(idim, ndim, perm, &axis_flipped); if (0 <= i && i < op_ndim) { shape = PyArray_DIM(op, i); if (shape != 1) { npy_intp stride = PyArray_STRIDE(op, i); - if (p < 0) { - /* If the perm entry is negative, flip the axis */ + if (axis_flipped) { NAD_STRIDES(axisdata)[iop] = -stride; baseoffset += stride*(shape-1); } @@ -1976,25 +1971,18 @@ npyiter_replace_axisdata(NpyIter *iter, int iop, } else { for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { - npy_int8 p; int i; + npy_bool axis_flipped; npy_intp shape; - /* Apply the perm to get the original axis */ - p = perm[idim]; - if (p < 0) { - i = op_ndim+p; - } - else { - i = op_ndim-p-1; - } + i = npyiter_undo_iter_axis_perm( + idim, orig_op_ndim, perm, &axis_flipped); if (i >= 0) { shape = PyArray_DIM(op, i); if (shape != 1) { npy_intp stride = PyArray_STRIDE(op, i); - if (p < 0) { - /* If the perm entry is negative, flip the axis */ + if (axis_flipped) { NAD_STRIDES(axisdata)[iop] = -stride; baseoffset += stride*(shape-1); } @@ -2488,7 +2476,7 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, stride = op_dtype->elsize; NpyIter_AxisData *axisdata; npy_intp sizeof_axisdata; - npy_intp i; + int i; PyArrayObject *ret; @@ -2521,16 +2509,9 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, if (op_axes != NULL) { for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { - npy_int8 p; /* Apply the perm to get the original axis */ - p = perm[idim]; - if (p < 0) { - i = op_axes[ndim+p]; - } - else { - i = op_axes[ndim-p-1]; - } + i = npyiter_undo_iter_axis_perm(idim, ndim, perm, NULL); if (i >= 0) { NPY_IT_DBG_PRINT3("Iterator: Setting allocated stride %d " @@ -2583,16 +2564,8 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, } else { for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { - npy_int8 p; - /* Apply the perm to get the original axis */ - p = perm[idim]; - if (p < 0) { - i = op_ndim + p; - } - else { - i = op_ndim - p - 1; - } + i = npyiter_undo_iter_axis_perm(idim, op_ndim, perm, NULL); if (i >= 0) { NPY_IT_DBG_PRINT3("Iterator: Setting allocated stride %d " diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index 5fb146026f74..82997e6feed4 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -301,6 +301,40 @@ struct NpyIter_AD { NIT_AXISDATA_SIZEOF(itflags, ndim, nop)*(ndim ? ndim : 1)) /* Internal helper functions shared between implementation files */ + +/** + * Undo the axis permutation of the iterator. When the operand has less + * dimensions then the iterator, this can return negative values for + * inserted (broadcast) dimensions. + * + * @param axis Axis for which to undo the iterator axis permutation. + * @param ndim If `op_axes` is being used, this is the iterator dimension, + * otherwise this is the operand dimension. + * @param perm he iterator axis permutation NIT_PERM(iter) + * @param axis_flipped Will be set to to true if this is a flipped axis and + * otherwise false. Can be NULL. + * @return The unpermuted axis. Without `op_axes` this is correct, with + * `op_axes` this indexes into `op_axes` (unpermuted iterator axis) + */ +static NPY_INLINE int +npyiter_undo_iter_axis_perm( + int axis, int ndim, const npy_int8 *perm, npy_bool *axis_flipped) +{ + npy_int8 p = perm[axis]; + /* The iterator treats axis reversed, thus adjust by ndim */ + npy_bool flipped = p < 0; + if (axis_flipped != NULL) { + *axis_flipped = flipped; + } + if (flipped) { + axis = ndim + p; + } + else { + axis = ndim - p - 1; + } + return axis; +} + NPY_NO_EXPORT void npyiter_coalesce_axes(NpyIter *iter); NPY_NO_EXPORT int From 407f62667cba244ab3d67efbebe52fd3e04b2924 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 13:47:22 -0500 Subject: [PATCH 0029/4003] MAINT: (nditer) move duplicated reduce axis check into helper func --- numpy/core/src/multiarray/nditer_constr.c | 97 ++++++++++------------- 1 file changed, 43 insertions(+), 54 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 4701e74d42a3..83ea52ba97b8 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1410,6 +1410,38 @@ check_mask_for_writemasked_reduction(NpyIter *iter, int iop) return 1; } +/* + * Check whether a reduction is OK based on the flags and the operand being + * readwrite. This path is deprecated, since usually only specific axes + * should be reduced. If axes are specified explicitely, the flag is + * unnecessary. + */ +static int +npyiter_check_reduce_ok_and_set_flags( + NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags) { + /* If it's writeable, this means a reduction */ + if (*op_itflags & NPY_OP_ITFLAG_WRITE) { + if (!(flags & NPY_ITER_REDUCE_OK)) { + PyErr_SetString(PyExc_ValueError, + "output operand requires a reduction, but reduction is" + "not enabled"); + return 0; + } + if (!(*op_itflags & NPY_OP_ITFLAG_READ)) { + PyErr_SetString(PyExc_ValueError, + "output operand requires a reduction, but is flagged as " + "write-only, not read-write"); + return 0; + } + NPY_IT_DBG_PRINT("Iterator: Indicating that a reduction is" + "occurring\n"); + + NIT_ITFLAGS(iter) |= NPY_ITFLAG_REDUCE; + *op_itflags |= NPY_OP_ITFLAG_REDUCE; + } + return 1; +} + /* * Fills in the AXISDATA for the 'nop' operands, broadcasting * the dimensionas as necessary. Also fills @@ -1621,51 +1653,20 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf if (op_flags[iop] & NPY_ITER_NO_BROADCAST) { goto operand_different_than_broadcast; } - /* If it's writeable, this means a reduction */ - if (op_itflags[iop] & NPY_OP_ITFLAG_WRITE) { - if (!(flags & NPY_ITER_REDUCE_OK)) { - PyErr_SetString(PyExc_ValueError, - "output operand requires a reduction, but " - "reduction is not enabled"); - return 0; - } - if (!(op_itflags[iop] & NPY_OP_ITFLAG_READ)) { - PyErr_SetString(PyExc_ValueError, - "output operand requires a reduction, but " - "is flagged as write-only, not " - "read-write"); - return 0; - } - NIT_ITFLAGS(iter) |= NPY_ITFLAG_REDUCE; - op_itflags[iop] |= NPY_OP_ITFLAG_REDUCE; + if (!npyiter_check_reduce_ok_and_set_flags( + iter, flags, &op_itflags[iop])) { + return 0; } } else { strides[iop] = PyArray_STRIDE(op_cur, i); } } - else if (bshape == 1) { - strides[iop] = 0; - } else { strides[iop] = 0; - /* If it's writeable, this means a reduction */ - if (op_itflags[iop] & NPY_OP_ITFLAG_WRITE) { - if (!(flags & NPY_ITER_REDUCE_OK)) { - PyErr_SetString(PyExc_ValueError, - "output operand requires a reduction, but " - "reduction is not enabled"); - return 0; - } - if (!(op_itflags[iop] & NPY_OP_ITFLAG_READ)) { - PyErr_SetString(PyExc_ValueError, - "output operand requires a reduction, but " - "is flagged as write-only, not " - "read-write"); - return 0; - } - NIT_ITFLAGS(iter) |= NPY_ITFLAG_REDUCE; - op_itflags[iop] |= NPY_OP_ITFLAG_REDUCE; + if (!npyiter_check_reduce_ok_and_set_flags( + iter, flags, &op_itflags[iop])) { + return 0; } } } @@ -2536,27 +2537,15 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, if (shape == NULL) { /* * If deleting this axis produces a reduction, but - * reduction wasn't enabled, throw an error + * reduction wasn't enabled, throw an error. + * NOTE: We currently always allow new-axis if the iteration + * size is 1 (thus allowing broadcasting sometimes). */ if (NAD_SHAPE(axisdata) != 1) { - if (!(flags & NPY_ITER_REDUCE_OK)) { - PyErr_SetString(PyExc_ValueError, - "output requires a reduction, but " - "reduction is not enabled"); - return NULL; - } - if (!((*op_itflags) & NPY_OP_ITFLAG_READ)) { - PyErr_SetString(PyExc_ValueError, - "output requires a reduction, but " - "is flagged as write-only, not read-write"); + if (!npyiter_check_reduce_ok_and_set_flags( + iter, flags, op_itflags)) { return NULL; } - - NPY_IT_DBG_PRINT("Iterator: Indicating that a " - "reduction is occurring\n"); - /* Indicate that a reduction is occurring */ - NIT_ITFLAGS(iter) |= NPY_ITFLAG_REDUCE; - (*op_itflags) |= NPY_OP_ITFLAG_REDUCE; } } } From f31aeefd7ff3ba5bd953a1e082226d08eb837a43 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 15:04:37 -0500 Subject: [PATCH 0030/4003] MAINT: Simplify signature of npyiter_replace_axisdata --- numpy/core/src/multiarray/nditer_constr.c | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 83ea52ba97b8..894d68d888c9 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -58,10 +58,9 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf const npy_uint32 *op_flags, int **op_axes, npy_intp const *itershape); static void -npyiter_replace_axisdata(NpyIter *iter, int iop, - PyArrayObject *op, - int op_ndim, char *op_dataptr, - int *op_axes); +npyiter_replace_axisdata( + NpyIter *iter, int iop, PyArrayObject *op, + int orig_op_ndim, const int *op_axes); static void npyiter_compute_index_strides(NpyIter *iter, npy_uint32 flags); static void @@ -1922,14 +1921,14 @@ operand_different_than_broadcast: { * array. */ static void -npyiter_replace_axisdata(NpyIter *iter, int iop, - PyArrayObject *op, - int op_ndim, char *op_dataptr, - int *op_axes) +npyiter_replace_axisdata( + NpyIter *iter, int iop, PyArrayObject *op, + int orig_op_ndim, const int *op_axes) { npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int nop = NIT_NOP(iter); + char *op_dataptr = PyArray_DATA(op); NpyIter_AxisData *axisdata0, *axisdata; npy_intp sizeof_axisdata; @@ -2810,8 +2809,8 @@ npyiter_allocate_arrays(NpyIter *iter, * Now we need to replace the pointers and strides with values * from the new array. */ - npyiter_replace_axisdata(iter, iop, op[iop], ondim, - PyArray_DATA(op[iop]), op_axes ? op_axes[iop] : NULL); + npyiter_replace_axisdata(iter, iop, op[iop], ndim, + op_axes ? op_axes[iop] : NULL); /* * New arrays are guaranteed true-aligned, but copy/cast code @@ -2852,8 +2851,7 @@ npyiter_allocate_arrays(NpyIter *iter, * Now we need to replace the pointers and strides with values * from the temporary array. */ - npyiter_replace_axisdata(iter, iop, op[iop], 0, - PyArray_DATA(op[iop]), NULL); + npyiter_replace_axisdata(iter, iop, op[iop], 0, NULL); /* * New arrays are guaranteed true-aligned, but copy/cast code @@ -2925,7 +2923,7 @@ npyiter_allocate_arrays(NpyIter *iter, * from the temporary array. */ npyiter_replace_axisdata(iter, iop, op[iop], ondim, - PyArray_DATA(op[iop]), op_axes ? op_axes[iop] : NULL); + op_axes ? op_axes[iop] : NULL); /* * New arrays are guaranteed true-aligned, but copy/cast code From 07ea525dd627ee593591ae18c6d6599d943ca116 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 16:01:51 -0500 Subject: [PATCH 0031/4003] MAINT: Simplify strides and axis check in npyiter_new_temp_array The stride calculation and axis check redid some of the work finding the number of dimensions again. This simplifies the logic and clarifies an error message a bit. --- numpy/core/src/multiarray/nditer_constr.c | 124 +++++++++++----------- numpy/core/tests/test_nditer.py | 15 +++ 2 files changed, 75 insertions(+), 64 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 894d68d888c9..2937c3f8533b 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -2469,11 +2469,12 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, { npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); + int used_op_ndim; int nop = NIT_NOP(iter); npy_int8 *perm = NIT_PERM(iter); - npy_intp new_shape[NPY_MAXDIMS], strides[NPY_MAXDIMS], - stride = op_dtype->elsize; + npy_intp new_shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; + npy_intp stride = op_dtype->elsize; NpyIter_AxisData *axisdata; npy_intp sizeof_axisdata; int i; @@ -2503,11 +2504,12 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, sizeof_axisdata = NIT_AXISDATA_SIZEOF(itflags, ndim, nop); /* Initialize the strides to invalid values */ - for (i = 0; i < NPY_MAXDIMS; ++i) { + for (i = 0; i < op_ndim; ++i) { strides[i] = NPY_MAX_INTP; } if (op_axes != NULL) { + used_op_ndim = 0; for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { /* Apply the perm to get the original axis */ @@ -2517,14 +2519,18 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, NPY_IT_DBG_PRINT3("Iterator: Setting allocated stride %d " "for iterator dimension %d to %d\n", (int)i, (int)idim, (int)stride); + used_op_ndim += 1; strides[i] = stride; if (shape == NULL) { new_shape[i] = NAD_SHAPE(axisdata); stride *= new_shape[i]; if (i >= ndim) { - PyErr_SetString(PyExc_ValueError, + PyErr_Format(PyExc_ValueError, "automatically allocated output array " - "specified with an inconsistent axis mapping"); + "specified with an inconsistent axis mapping; " + "the axis mapping cannot include dimension %d " + "which is too large for the iterator dimension " + "of %d.", i, ndim); return NULL; } } @@ -2551,6 +2557,7 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, } } else { + used_op_ndim = ndim; for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { /* Apply the perm to get the original axis */ i = npyiter_undo_iter_axis_perm(idim, op_ndim, perm, NULL); @@ -2571,73 +2578,57 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, } } - /* - * If custom axes were specified, some dimensions may not have been used. - * Add the REDUCE itflag if this creates a reduction situation. - */ if (shape == NULL) { - /* Ensure there are no dimension gaps in op_axes, and find op_ndim */ - op_ndim = ndim; - if (op_axes != NULL) { - for (i = 0; i < ndim; ++i) { - if (strides[i] == NPY_MAX_INTP) { - if (op_ndim == ndim) { - op_ndim = i; - } - } - /* - * If there's a gap in the array's dimensions, it's an error. - * For example, op_axes of [0,2] for the automatically - * allocated output. - */ - else if (op_ndim != ndim) { - PyErr_SetString(PyExc_ValueError, - "automatically allocated output array " - "specified with an inconsistent axis mapping"); - return NULL; - } + /* If shape was NULL, use the shape we calculated */ + op_ndim = used_op_ndim; + shape = new_shape; + /* + * If op_axes [0, 2] is specified, there will a place in the strides + * array where the value is not set. + */ + for (i = 0; i < op_ndim; i++) { + if (strides[i] == NPY_MAX_INTP) { + PyErr_Format(PyExc_ValueError, + "automatically allocated output array " + "specified with an inconsistent axis mapping; " + "the axis mapping is missing an entry for " + "dimension %d.", i); + return NULL; } } } - else { - for (i = 0; i < op_ndim; ++i) { - if (strides[i] == NPY_MAX_INTP) { - npy_intp factor, new_strides[NPY_MAXDIMS], - itemsize; - - /* Fill in the missing strides in C order */ - factor = 1; - itemsize = op_dtype->elsize; - for (i = op_ndim-1; i >= 0; --i) { - if (strides[i] == NPY_MAX_INTP) { - new_strides[i] = factor * itemsize; - factor *= shape[i]; - } - } - - /* - * Copy the missing strides, and multiply the existing strides - * by the calculated factor. This way, the missing strides - * are tighter together in memory, which is good for nested - * loops. - */ - for (i = 0; i < op_ndim; ++i) { - if (strides[i] == NPY_MAX_INTP) { - strides[i] = new_strides[i]; - } - else { - strides[i] *= factor; - } - } + else if (used_op_ndim < op_ndim) { + /* + * If custom axes were specified, some dimensions may not have + * been used. These are additional axes which are ignored in the + * iterator but need to be handled here. + */ + npy_intp factor, itemsize, new_strides[NPY_MAXDIMS]; - break; + /* Fill in the missing strides in C order */ + factor = 1; + itemsize = op_dtype->elsize; + for (i = op_ndim-1; i >= 0; --i) { + if (strides[i] == NPY_MAX_INTP) { + new_strides[i] = factor * itemsize; + factor *= shape[i]; } } - } - /* If shape was NULL, set it to the shape we calculated */ - if (shape == NULL) { - shape = new_shape; + /* + * Copy the missing strides, and multiply the existing strides + * by the calculated factor. This way, the missing strides + * are tighter together in memory, which is good for nested + * loops. + */ + for (i = 0; i < op_ndim; ++i) { + if (strides[i] == NPY_MAX_INTP) { + strides[i] = new_strides[i]; + } + else { + strides[i] *= factor; + } + } } /* Allocate the temporary array */ @@ -2650,6 +2641,11 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, /* Double-check that the subtype didn't mess with the dimensions */ if (subtype != &PyArray_Type) { + /* + * TODO: the dtype could have a subarray, which adds new dimensions + * to `ret`, that should typically be fine, but will break + * in this branch. + */ if (PyArray_NDIM(ret) != op_ndim || !PyArray_CompareLists(shape, PyArray_DIMS(ret), op_ndim)) { PyErr_SetString(PyExc_RuntimeError, diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index c106c528dd21..7b3c3a40d73d 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1520,6 +1520,13 @@ def test_iter_allocate_output_errors(): [['readonly'], ['writeonly', 'allocate']], op_dtypes=[None, np.dtype('f4')], op_axes=[None, [0, 2, 1, 0]]) + # Not all axes may be specified if a reduction. If there is a hole + # in op_axes, this is an error. + a = arange(24, dtype='i4').reshape(2, 3, 4) + assert_raises(ValueError, nditer, [a, None], ["reduce_ok"], + [['readonly'], ['readwrite', 'allocate']], + op_dtypes=[None, np.dtype('f4')], + op_axes=[None, [0, np.newaxis, 2]]) def test_iter_remove_axis(): a = arange(24).reshape(2, 3, 4) @@ -2661,6 +2668,14 @@ def test_iter_allocated_array_dtypes(): b[1] = a + 1 assert_equal(it.operands[1], [[0, 2], [2, 4], [19, 21]]) + # Check the same (less sensitive) thing when `op_axes` with -1 is given. + it = np.nditer(([[1, 3, 20]], None), op_dtypes=[None, ('i4', (2,))], + flags=["reduce_ok"], op_axes=[None, (-1, 0)]) + for a, b in it: + b[0] = a - 1 + b[1] = a + 1 + assert_equal(it.operands[1], [[0, 2], [2, 4], [19, 21]]) + # Make sure this works for scalars too it = np.nditer((10, 2, None), op_dtypes=[None, None, ('i4', (2, 2))]) for a, b, c in it: From 7f8dc86765f6f52e45126c3d7c80ac00eb01d271 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 16:26:05 -0500 Subject: [PATCH 0032/4003] ENH: Allow specifying reduction axis in nditer/NpyIter A reduction axis can be a broadcast axis (-1), which does not exist. Or an existing axis which then must have a length of 1, or will be allocated with a length of 1. This allows the correct allocation of reduction outputs with axes of size 1 (instead of only newaxis/-1), i.e. it allows supporting the result output with `keepdims=True` in reductions. It can also be used by generalized ufuncs to specific that one operand does not need to iterate a core dimensions. In the case of generalized ufuncs, this is handled like a reduce dimensions, but that dimension is later removed. This means that the operand can be specified without read-write. It mainly is necessary to allow normal broadcasting for the axes _not_ marked as reduce axis. In most cases however, an operand should never use broadcasting when marking axes as reduction. --- numpy/core/src/multiarray/common.h | 9 +++ numpy/core/src/multiarray/nditer_constr.c | 95 ++++++++++++++++++++--- 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 4913eb202f22..eb6eef24035f 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -342,5 +342,14 @@ NPY_NO_EXPORT PyArrayObject * new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, int nd, npy_intp dimensions[], int typenum, PyArrayObject **result); + +/* + * Used to indicate a broadcast axis, see also `npyiter_get_op_axis` in + * `nditer_constr.c`. This may be the preferred API for reduction axes + * probably. So we should consider making this public either as a macro or + * function (so that the way we flag the axis can be changed). + */ +#define NPY_ITER_REDUCTION_AXIS(axis) (axis + (NPY_MAX_INT >> 1)) + #endif diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 2937c3f8533b..7bc40f03aedb 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -57,6 +57,8 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf char **op_dataptr, const npy_uint32 *op_flags, int **op_axes, npy_intp const *itershape); +static NPY_INLINE int +npyiter_get_op_axis(int axis, npy_bool *reduction_axis); static void npyiter_replace_axisdata( NpyIter *iter, int iop, PyArrayObject *op, @@ -825,7 +827,8 @@ npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, if (axes != NULL) { memset(axes_dupcheck, 0, NPY_MAXDIMS); for (idim = 0; idim < oa_ndim; ++idim) { - int i = axes[idim]; + int i = npyiter_get_op_axis(axes[idim], NULL); + if (i >= 0) { if (i >= NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, @@ -1441,6 +1444,27 @@ npyiter_check_reduce_ok_and_set_flags( return 1; } +/** + * Removes the (additive) NPY_ITER_REDUCTION_AXIS indication and sets + * is_forced_broadcast to 1 if it is set. Otherwise to 0. + * + * @param axis The op_axes[i] to normalize. + * @param reduction_axis Output 1 if a reduction axis, otherwise 0. + * @returns The normalized axis (without reduce axis flag). + */ +static NPY_INLINE int +npyiter_get_op_axis(int axis, npy_bool *reduction_axis) { + npy_bool forced_broadcast = axis >= NPY_ITER_REDUCTION_AXIS(-1); + + if (reduction_axis != NULL) { + *reduction_axis = forced_broadcast; + } + if (forced_broadcast) { + return axis - NPY_ITER_REDUCTION_AXIS(0); + } + return axis; +} + /* * Fills in the AXISDATA for the 'nop' operands, broadcasting * the dimensionas as necessary. Also fills @@ -1515,7 +1539,8 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf else { int *axes = op_axes[iop]; for (idim = 0; idim < ndim; ++idim) { - int i = axes[idim]; + int i = npyiter_get_op_axis(axes[idim], NULL); + if (i >= 0) { if (i < ondim) { npy_intp bshape = broadcast_shape[idim]; @@ -1642,9 +1667,37 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf } else { int *axes = op_axes[iop]; - int i = axes[ndim-idim-1]; - if (i >= 0) { - if (bshape == 1 || op_cur == NULL) { + npy_bool reduction_axis; + int i; + i = npyiter_get_op_axis(axes[ndim - idim - 1], &reduction_axis); + + if (reduction_axis) { + /* This is explicitly a reduction axis */ + strides[iop] = 0; + NIT_ITFLAGS(iter) |= NPY_ITFLAG_REDUCE; + op_itflags[iop] |= NPY_OP_ITFLAG_REDUCE; + + if (NPY_UNLIKELY((i >= 0) && (op_cur != NULL) && + (PyArray_DIM(op_cur, i) != 1))) { + PyErr_Format(PyExc_ValueError, + "operand was set up as a reduction along axis " + "%d, but the length of the axis is %zd " + "(it has to be 1)", + i, (Py_ssize_t)PyArray_DIM(op_cur, i)); + return 0; + } + } + else if (bshape == 1) { + /* + * If the full iterator shape is 1, zero always works. + * NOTE: We thus always allow broadcast dimensions (i = -1) + * if the shape is 1. + */ + strides[iop] = 0; + } + else if (i >= 0) { + if (op_cur == NULL) { + /* stride is filled later, shape will match `bshape` */ strides[iop] = 0; } else if (PyArray_DIM(op_cur, i) == 1) { @@ -1775,7 +1828,7 @@ broadcast_error: { if (axes != NULL) { for (idim = 0; idim < ndim; ++idim) { - npy_intp i = axes[idim]; + int i = npyiter_get_op_axis(axes[idim], NULL); if (i >= 0 && i < PyArray_NDIM(op[iop])) { remdims[idim] = PyArray_DIM(op[iop], i); @@ -1954,7 +2007,9 @@ npyiter_replace_axisdata( /* Apply perm to get the original axis, and check if its flipped */ i = npyiter_undo_iter_axis_perm(idim, ndim, perm, &axis_flipped); - if (0 <= i && i < op_ndim) { + i = npyiter_get_op_axis(op_axes[i], NULL); + assert(i < orig_op_ndim); + if (i >= 0) { shape = PyArray_DIM(op, i); if (shape != 1) { npy_intp stride = PyArray_STRIDE(op, i); @@ -2511,9 +2566,11 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, if (op_axes != NULL) { used_op_ndim = 0; for (idim = 0; idim < ndim; ++idim, NIT_ADVANCE_AXISDATA(axisdata, 1)) { + npy_bool reduction_axis; /* Apply the perm to get the original axis */ i = npyiter_undo_iter_axis_perm(idim, ndim, perm, NULL); + i = npyiter_get_op_axis(op_axes[i], &reduction_axis); if (i >= 0) { NPY_IT_DBG_PRINT3("Iterator: Setting allocated stride %d " @@ -2522,7 +2579,13 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, used_op_ndim += 1; strides[i] = stride; if (shape == NULL) { - new_shape[i] = NAD_SHAPE(axisdata); + if (reduction_axis) { + /* reduction axes always have a length of 1 */ + new_shape[i] = 1; + } + else { + new_shape[i] = NAD_SHAPE(axisdata); + } stride *= new_shape[i]; if (i >= ndim) { PyErr_Format(PyExc_ValueError, @@ -2536,6 +2599,9 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, } else { stride *= shape[i]; + if (reduction_axis) { + assert(shape[i] == 1); + } } } else { @@ -2546,7 +2612,7 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, * NOTE: We currently always allow new-axis if the iteration * size is 1 (thus allowing broadcasting sometimes). */ - if (NAD_SHAPE(axisdata) != 1) { + if (!reduction_axis && NAD_SHAPE(axisdata) != 1) { if (!npyiter_check_reduce_ok_and_set_flags( iter, flags, op_itflags)) { return NULL; @@ -2782,16 +2848,21 @@ npyiter_allocate_arrays(NpyIter *iter, if (op[iop] == NULL) { PyArrayObject *out; PyTypeObject *op_subtype; - int ondim = ndim; /* Check whether the subtype was disabled */ op_subtype = (op_flags[iop] & NPY_ITER_NO_SUBTYPE) ? &PyArray_Type : subtype; - /* Allocate the output array */ + /* + * Allocate the output array. + * + * Note that here, ndim is always correct if no op_axes was given + * (but the actual dimension of op can be larger). If op_axes + * is given, ndim is not actually used. + */ out = npyiter_new_temp_array(iter, op_subtype, flags, &op_itflags[iop], - ondim, + ndim, NULL, op_dtype[iop], op_axes ? op_axes[iop] : NULL); From 87cb35f371c5711ea20979b9e2569ca544b237e6 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 16:33:04 -0500 Subject: [PATCH 0033/4003] MAINT: Use reduction axis marker for einsum --- numpy/core/src/multiarray/einsum.c.src | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/einsum.c.src b/numpy/core/src/multiarray/einsum.c.src index 1cc557825867..b914e5bb3c04 100644 --- a/numpy/core/src/multiarray/einsum.c.src +++ b/numpy/core/src/multiarray/einsum.c.src @@ -2740,7 +2740,7 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop, op_axes[nop][idim] = idim; } for (idim = ndim_output; idim < ndim_iter; ++idim) { - op_axes[nop][idim] = -1; + op_axes[nop][idim] = NPY_ITER_REDUCTION_AXIS(-1); } /* Set the iterator per-op flags */ @@ -2753,13 +2753,11 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop, op_flags[nop] = NPY_ITER_READWRITE| NPY_ITER_NBO| NPY_ITER_ALIGNED| - NPY_ITER_ALLOCATE| - NPY_ITER_NO_BROADCAST; + NPY_ITER_ALLOCATE; iter_flags = NPY_ITER_EXTERNAL_LOOP| NPY_ITER_BUFFERED| NPY_ITER_DELAY_BUFALLOC| NPY_ITER_GROWINNER| - NPY_ITER_REDUCE_OK| NPY_ITER_REFS_OK| NPY_ITER_ZEROSIZE_OK; if (out != NULL) { From b5c2f94df50bab697eaa7d9727a4f3b44f89d329 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 16:34:29 -0500 Subject: [PATCH 0034/4003] BUG: Avoid incorrect broadcasts on non-core outputs in gufuncs Mark the core dimensions as reduce axis. This allows them to be freely broadcast (basically ignored for most practical purposes) while other axes are normally broadcast even though the operand is read-only or write-only. If we were to use read-write operands with `REDUCE_OK` these additional dimensions are currently simply absored as reduction dimensions... (I hope this is understandable, please see the issues/test for an example...) Fixes gh-15139, replaces gh-15142 Co-Authored-By: Marten van Kerkwijk --- numpy/core/src/umath/ufunc_object.c | 13 ++++++++----- numpy/core/tests/test_ufunc.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index c57199c79bfd..19876d641a46 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2697,9 +2697,13 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, } } - /* Any output core dimensions shape should be ignored */ + /* + * Any output core dimensions shape should be ignored, so we add + * it as a Reduce dimension (which can be broadcast with the rest). + * These will be removed before the actual iteration for gufuncs. + */ for (idim = broadcast_ndim; idim < iter_ndim; ++idim) { - op_axes_arrays[i][idim] = -1; + op_axes_arrays[i][idim] = NPY_ITER_REDUCTION_AXIS(-1); } /* Except for when it belongs to this output */ @@ -2771,7 +2775,7 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, */ _ufunc_setup_flags(ufunc, NPY_ITER_COPY | NPY_UFUNC_DEFAULT_INPUT_FLAGS, NPY_ITER_UPDATEIFCOPY | - NPY_ITER_READWRITE | + NPY_ITER_WRITEONLY | NPY_UFUNC_DEFAULT_OUTPUT_FLAGS, op_flags); /* For the generalized ufunc, we get the loop right away too */ @@ -2820,7 +2824,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, iter_flags = ufunc->iter_flags | NPY_ITER_MULTI_INDEX | NPY_ITER_REFS_OK | - NPY_ITER_REDUCE_OK | NPY_ITER_ZEROSIZE_OK | NPY_ITER_COPY_IF_OVERLAP; @@ -3646,7 +3649,7 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, result = PyUFunc_ReduceWrapper(arr, out, wheremask, dtype, dtype, NPY_UNSAFE_CASTING, axis_flags, reorderable, - keepdims, 0, + keepdims, initial, reduce_loop, ufunc, buffersize, ufunc_name, errormask); diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index abdaeeb93e2c..e473652931bf 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -610,7 +610,17 @@ def test_broadcast(self): warnings.simplefilter("always") u += v assert_equal(len(w), 1) - assert_(x[0,0] != u[0, 0]) + assert_(x[0, 0] != u[0, 0]) + + # Output reduction should not be allowed. + # See gh-15139 + a = np.arange(6).reshape(3, 2) + b = np.ones(2) + out = np.empty(()) + assert_raises(ValueError, umt.inner1d, a, b, out) + out2 = np.empty(3) + c = umt.inner1d(a, b, out2) + assert_(c is out2) def test_type_cast(self): msg = "type cast" @@ -942,7 +952,10 @@ def test_cross1d(self): assert_array_equal(result, np.vstack((np.zeros(3), a[2], -a[1]))) assert_raises(ValueError, umt.cross1d, np.eye(4), np.eye(4)) assert_raises(ValueError, umt.cross1d, a, np.arange(4.)) + # Wrong output core dimension. assert_raises(ValueError, umt.cross1d, a, np.arange(3.), np.zeros((3, 4))) + # Wrong output broadcast dimension (see gh-15139). + assert_raises(ValueError, umt.cross1d, a, np.arange(3.), np.zeros(3)) def test_can_ignore_signature(self): # Comparing the effects of ? in signature: From 729b9ca93c43ac3442484929bd52574da8bee4c7 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 16:59:51 -0500 Subject: [PATCH 0035/4003] BUG,MAINT: Simplify reduction result creation using reduce axis The new reduce axis marker for nditer/NpyIter allows to move the allocation logic necessary to allocate the reduction result to the nditer, thus simplifying the logic here. This leads to some other related cleanups of the result initialization. The subok flag was removed, since it was always set to 0 and the iterator rejects subclasses. --- numpy/core/src/umath/reduction.c | 510 +++++++++---------------------- numpy/core/src/umath/reduction.h | 5 +- numpy/core/tests/test_ufunc.py | 32 +- 3 files changed, 160 insertions(+), 387 deletions(-) diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index 79adb0051fe8..bcfbdd9547f7 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -25,235 +25,6 @@ #include "reduction.h" #include "extobj.h" /* for _check_ufunc_fperr */ -/* - * Allocates a result array for a reduction operation, with - * dimensions matching 'arr' except set to 1 with 0 stride - * wherever axis_flags is True. Dropping the reduction axes - * from the result must be done later by the caller once the - * computation is complete. - * - * This function always allocates a base class ndarray. - * - * If 'dtype' isn't NULL, this function steals its reference. - */ -static PyArrayObject * -allocate_reduce_result(PyArrayObject *arr, const npy_bool *axis_flags, - PyArray_Descr *dtype, int subok) -{ - npy_intp strides[NPY_MAXDIMS], stride; - npy_intp shape[NPY_MAXDIMS], *arr_shape = PyArray_DIMS(arr); - npy_stride_sort_item strideperm[NPY_MAXDIMS]; - int idim, ndim = PyArray_NDIM(arr); - - if (dtype == NULL) { - dtype = PyArray_DTYPE(arr); - Py_INCREF(dtype); - } - - PyArray_CreateSortedStridePerm(PyArray_NDIM(arr), - PyArray_STRIDES(arr), strideperm); - - /* Build the new strides and shape */ - stride = dtype->elsize; - if (ndim) { - memcpy(shape, arr_shape, ndim * sizeof(shape[0])); - } - for (idim = ndim-1; idim >= 0; --idim) { - npy_intp i_perm = strideperm[idim].perm; - if (axis_flags[i_perm]) { - strides[i_perm] = 0; - shape[i_perm] = 1; - } - else { - strides[i_perm] = stride; - stride *= shape[i_perm]; - } - } - - /* Finally, allocate the array */ - return (PyArrayObject *)PyArray_NewFromDescr( - subok ? Py_TYPE(arr) : &PyArray_Type, - dtype, ndim, shape, strides, - NULL, 0, subok ? (PyObject *)arr : NULL); -} - -/* - * Conforms an output parameter 'out' to have 'ndim' dimensions - * with dimensions of size one added in the appropriate places - * indicated by 'axis_flags'. - * - * The return value is a view into 'out'. - */ -static PyArrayObject * -conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, - PyArrayObject *out, int keepdims, const char *funcname, - int need_copy) -{ - /* unpack shape information */ - int const ndim = PyArray_NDIM(in); - npy_intp const *shape_in = PyArray_DIMS(in); - - int const ndim_out = PyArray_NDIM(out); - npy_intp const *strides_out = PyArray_STRIDES(out); - npy_intp const *shape_out = PyArray_DIMS(out); - - /* - * If the 'keepdims' parameter is true, do a simpler validation and - * return a new reference to 'out'. - */ - if (keepdims) { - if (PyArray_NDIM(out) != ndim) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "has the wrong number of dimensions (must match " - "the operand's when keepdims=True)", funcname); - return NULL; - } - - for (int idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - if (shape_out[idim] != 1) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "has a reduction dimension not equal to one " - "(required when keepdims=True)", funcname); - return NULL; - } - } - else { - if (shape_out[idim] != shape_in[idim]) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "has a non-reduction dimension not equal to " - "the input one.", funcname); - return NULL; - } - } - - } - - Py_INCREF(out); - return out; - } - - /* Construct the strides and shape */ - npy_intp strides[NPY_MAXDIMS], shape[NPY_MAXDIMS]; - int idim_out = 0; - for (int idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - strides[idim] = 0; - shape[idim] = 1; - } - else { - if (idim_out >= ndim_out) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "does not have enough dimensions", funcname); - return NULL; - } - if (shape_out[idim_out] != shape_in[idim]) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "has a non-reduction dimension not equal to " - "the input one.", funcname); - return NULL; - } - strides[idim] = strides_out[idim_out]; - shape[idim] = shape_out[idim_out]; - ++idim_out; - } - } - - if (idim_out != ndim_out) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "has too many dimensions", funcname); - return NULL; - } - - /* Allocate the view */ - PyArray_Descr *dtype = PyArray_DESCR(out); - Py_INCREF(dtype); - - PyArrayObject_fields *ret = (PyArrayObject_fields *)PyArray_NewFromDescrAndBase( - &PyArray_Type, dtype, - ndim, shape, strides, PyArray_DATA(out), - PyArray_FLAGS(out), NULL, (PyObject *)out); - if (ret == NULL) { - return NULL; - } - - if (need_copy) { - PyArrayObject *ret_copy = (PyArrayObject *)PyArray_NewLikeArray( - (PyArrayObject *)ret, NPY_ANYORDER, NULL, 0); - if (ret_copy == NULL) { - Py_DECREF(ret); - return NULL; - } - - if (PyArray_CopyInto(ret_copy, (PyArrayObject *)ret) != 0) { - Py_DECREF(ret); - Py_DECREF(ret_copy); - return NULL; - } - - if (PyArray_SetWritebackIfCopyBase(ret_copy, (PyArrayObject *)ret) < 0) { - Py_DECREF(ret_copy); - return NULL; - } - - return ret_copy; - } - else { - return (PyArrayObject *)ret; - } -} - -/* - * Creates a result for reducing 'operand' along the axes specified - * in 'axis_flags'. If 'dtype' isn't NULL, this function steals a - * reference to 'dtype'. - * - * If 'out' isn't NULL, this function creates a view conforming - * to the number of dimensions of 'operand', adding a singleton dimension - * for each reduction axis specified. In this case, 'dtype' is ignored - * (but its reference is still stolen), and the caller must handle any - * type conversion/validity check for 'out' - * - * If 'subok' is true, creates a result with the subtype of 'operand', - * otherwise creates on with the base ndarray class. - * - * If 'out' is NULL, it allocates a new array whose shape matches that of - * 'operand', except for at the reduction axes. If 'dtype' is NULL, the dtype - * of 'operand' is used for the result. - */ -NPY_NO_EXPORT PyArrayObject * -PyArray_CreateReduceResult(PyArrayObject *operand, PyArrayObject *out, - PyArray_Descr *dtype, npy_bool *axis_flags, - int keepdims, int subok, - const char *funcname) -{ - PyArrayObject *result; - - if (out == NULL) { - /* This function steals the reference to 'dtype' */ - result = allocate_reduce_result(operand, axis_flags, dtype, subok); - } - else { - int need_copy = 0; - - if (solve_may_share_memory(operand, out, 1) != 0) { - need_copy = 1; - } - - /* Steal the dtype reference */ - Py_XDECREF(dtype); - result = conform_reduce_result(operand, axis_flags, - out, keepdims, funcname, need_copy); - } - - return result; -} /* * Count the number of dimensions selected in 'axis_flags' @@ -275,18 +46,15 @@ count_axes(int ndim, const npy_bool *axis_flags) /* * This function initializes a result array for a reduction operation * which has no identity. This means it needs to copy the first element - * it sees along the reduction axes to result, then return a view of - * the operand which excludes that element. + * it sees along the reduction axes to result. * * If a reduction has an identity, such as 0 or 1, the result should be - * initialized by calling PyArray_AssignZero(result, NULL, NULL) or - * PyArray_AssignOne(result, NULL, NULL), because this function raises an - * exception when there are no elements to reduce (which appropriate iff the - * reduction operation has no identity). + * fully initialized to the identity, because this function raises an + * exception when there are no elements to reduce (which is appropriate if, + * and only if, the reduction operation has no identity). * * This means it copies the subarray indexed at zero along each reduction axis - * into 'result', then returns a view into 'operand' excluding those copied - * elements. + * into 'result'. * * result : The array into which the result is computed. This must have * the same number of dimensions as 'operand', but for each @@ -294,105 +62,83 @@ count_axes(int ndim, const npy_bool *axis_flags) * operand : The array being reduced. * axis_flags : An array of boolean flags, one for each axis of 'operand'. * When a flag is True, it indicates to reduce along that axis. - * out_skip_first_count : This gets populated with the number of first-visit - * elements that should be skipped during the - * iteration loop. * funcname : The name of the reduction operation, for the purpose of * better quality error messages. For example, "numpy.max" * would be a good name for NumPy's max function. * - * Returns a view which contains the remaining elements on which to do - * the reduction. + * Returns -1 if an error occurred, and otherwise the reduce arrays size, + * which is the number of elements already initialized. */ -NPY_NO_EXPORT PyArrayObject * -PyArray_InitializeReduceResult( +NPY_NO_EXPORT int +PyArray_CopyInitialReduceValues( PyArrayObject *result, PyArrayObject *operand, - const npy_bool *axis_flags, - npy_intp *out_skip_first_count, const char *funcname) + const npy_bool *axis_flags, const char *funcname, + int keepdims) { - npy_intp *strides, *shape, shape_orig[NPY_MAXDIMS]; + npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; + npy_intp *shape_orig = PyArray_SHAPE(operand); + npy_intp *strides_orig = PyArray_STRIDES(operand); PyArrayObject *op_view = NULL; - int idim, ndim, nreduce_axes; - ndim = PyArray_NDIM(operand); - - /* Default to no skipping first-visit elements in the iteration */ - *out_skip_first_count = 0; - - /* Take a view into 'operand' which we can modify. */ - op_view = (PyArrayObject *)PyArray_View(operand, NULL, &PyArray_Type); - if (op_view == NULL) { - return NULL; - } + int ndim = PyArray_NDIM(operand); /* - * Now copy the subarray of the first element along each reduction axis, - * then return a view to the rest. + * Copy the subarray of the first element along each reduction axis. * * Adjust the shape to only look at the first element along - * any of the reduction axes. We count the number of reduction axes - * at the same time. + * any of the reduction axes. If keepdims is False remove the axes + * entirely. */ - shape = PyArray_SHAPE(op_view); - nreduce_axes = 0; - if (ndim) { - memcpy(shape_orig, shape, ndim * sizeof(npy_intp)); - } - for (idim = 0; idim < ndim; ++idim) { + int idim_out = 0; + npy_intp size = 1; + for (int idim = 0; idim < ndim; idim++) { if (axis_flags[idim]) { - if (shape[idim] == 0) { + if (NPY_UNLIKELY(shape_orig[idim] == 0)) { PyErr_Format(PyExc_ValueError, - "zero-size array to reduction operation %s " - "which has no identity", - funcname); - Py_DECREF(op_view); - return NULL; + "zero-size array to reduction operation %s " + "which has no identity", funcname); + return -1; } - shape[idim] = 1; - ++nreduce_axes; + if (keepdims) { + shape[idim_out] = 1; + strides[idim_out] = 0; + idim_out++; + } + } + else { + size *= shape_orig[idim]; + shape[idim_out] = shape_orig[idim]; + strides[idim_out] = strides_orig[idim]; + idim_out++; } } - /* - * Copy the elements into the result to start. - */ - if (PyArray_CopyInto(result, op_view) < 0) { - Py_DECREF(op_view); - return NULL; + PyArray_Descr *descr = PyArray_DESCR(operand); + Py_INCREF(descr); + op_view = (PyArrayObject *)PyArray_NewFromDescr( + &PyArray_Type, descr, idim_out, shape, strides, + PyArray_DATA(operand), 0, NULL); + if (op_view == NULL) { + return -1; } /* - * If there is one reduction axis, adjust the view's - * shape to only look at the remaining elements + * Copy the elements into the result to start. */ - if (nreduce_axes == 1) { - strides = PyArray_STRIDES(op_view); - for (idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - shape[idim] = shape_orig[idim] - 1; - ((PyArrayObject_fields *)op_view)->data += strides[idim]; - } - } - } - /* If there are zero reduction axes, make the view empty */ - else if (nreduce_axes == 0) { - for (idim = 0; idim < ndim; ++idim) { - shape[idim] = 0; - } + int res = PyArray_CopyInto(result, op_view); + Py_DECREF(op_view); + if (res < 0) { + return -1; } + /* - * Otherwise iterate over the whole operand, but tell the inner loop - * to skip the elements we already copied by setting the skip_first_count. + * If there were no reduction axes, we would already be done here. + * Note that if there is only a single reduction axis, in principle the + * iteration could be set up more efficiently here by removing that + * axis before setting up the iterator (simplifying the iteration since + * `skip_first_count` (the returned size) can be set to 0). */ - else { - *out_skip_first_count = PyArray_SIZE(result); - - Py_DECREF(op_view); - Py_INCREF(operand); - op_view = operand; - } - - return op_view; + return size; } /* @@ -418,7 +164,7 @@ PyArray_InitializeReduceResult( * with size one. * subok : If true, the result uses the subclass of operand, otherwise * it is always a base class ndarray. - * identity : If Py_None, PyArray_InitializeReduceResult is used, otherwise + * identity : If Py_None, PyArray_CopyInitialReduceValues is used, otherwise * this value is used to initialize the result to * the reduction's unit. * loop : The loop which does the reduction. @@ -444,13 +190,12 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_CASTING casting, npy_bool *axis_flags, int reorderable, int keepdims, - int subok, PyObject *identity, PyArray_ReduceLoopFunc *loop, void *data, npy_intp buffersize, const char *funcname, int errormask) { - PyArrayObject *result = NULL, *op_view = NULL; + PyArrayObject *result = NULL; npy_intp skip_first_count = 0; /* Iterator parameters */ @@ -476,49 +221,10 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, return NULL; } - /* - * This either conforms 'out' to the ndim of 'operand', or allocates - * a new array appropriate for this reduction. - * - * A new array with WRITEBACKIFCOPY is allocated if operand and out have memory - * overlap. - */ - Py_INCREF(result_dtype); - result = PyArray_CreateReduceResult(operand, out, - result_dtype, axis_flags, - keepdims, subok, funcname); - if (result == NULL) { - goto fail; - } - - /* - * Initialize the result to the reduction unit if possible, - * otherwise copy the initial values and get a view to the rest. - */ - if (identity != Py_None) { - if (PyArray_FillWithScalar(result, identity) < 0) { - goto fail; - } - op_view = operand; - Py_INCREF(op_view); - } - else { - op_view = PyArray_InitializeReduceResult( - result, operand, axis_flags, &skip_first_count, funcname); - if (op_view == NULL) { - goto fail; - } - /* empty op_view signals no reduction; but 0-d arrays cannot be empty */ - if ((PyArray_SIZE(op_view) == 0) || (PyArray_NDIM(operand) == 0)) { - Py_DECREF(op_view); - op_view = NULL; - goto finish; - } - } /* Set up the iterator */ - op[0] = result; - op[1] = op_view; + op[0] = out; + op[1] = operand; op_dtypes[0] = result_dtype; op_dtypes[1] = operand_dtype; @@ -527,13 +233,16 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_ITER_GROWINNER | NPY_ITER_DONT_NEGATE_STRIDES | NPY_ITER_ZEROSIZE_OK | - NPY_ITER_REDUCE_OK | - NPY_ITER_REFS_OK; + NPY_ITER_REFS_OK | + NPY_ITER_DELAY_BUFALLOC | + NPY_ITER_COPY_IF_OVERLAP; op_flags[0] = NPY_ITER_READWRITE | NPY_ITER_ALIGNED | + NPY_ITER_ALLOCATE | NPY_ITER_NO_SUBTYPE; op_flags[1] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; + if (wheremask != NULL) { op[2] = wheremask; /* wheremask is guaranteed to be NPY_BOOL, so borrow its reference */ @@ -545,15 +254,84 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, op_flags[2] = NPY_ITER_READONLY; } + /* Set up result array axes mapping, operand and wheremask use default */ + int result_axes[NPY_MAXDIMS]; + int *op_axes[3] = {result_axes, NULL, NULL}; + + int curr_axis = 0; + for (int i = 0; i < PyArray_NDIM(operand); i++) { + if (axis_flags[i]) { + if (keepdims) { + result_axes[i] = NPY_ITER_REDUCTION_AXIS(curr_axis); + curr_axis++; + } + else { + result_axes[i] = NPY_ITER_REDUCTION_AXIS(-1); + } + } + else { + result_axes[i] = curr_axis; + curr_axis++; + } + } + if (out != NULL) { + /* NpyIter does not raise a good error message in this common case. */ + if (NPY_UNLIKELY(curr_axis != PyArray_NDIM(out))) { + if (keepdims) { + PyErr_Format(PyExc_ValueError, + "output parameter for reduction operation %s has the " + "wrong number of dimensions: Found %d but expected %d " + "(must match the operand's when keepdims=True)", + funcname, PyArray_NDIM(out), curr_axis); + } + else { + PyErr_Format(PyExc_ValueError, + "output parameter for reduction operation %s has the " + "wrong number of dimensions: Found %d but expected %d", + funcname, PyArray_NDIM(out), curr_axis); + } + goto fail; + } + } + iter = NpyIter_AdvancedNew(wheremask == NULL ? 2 : 3, op, flags, NPY_KEEPORDER, casting, op_flags, op_dtypes, - -1, NULL, NULL, buffersize); + PyArray_NDIM(operand), op_axes, NULL, buffersize); if (iter == NULL) { goto fail; } + result = NpyIter_GetOperandArray(iter)[0]; + + /* + * Initialize the result to the reduction unit if possible, + * otherwise copy the initial values and get a view to the rest. + */ + + if (identity != Py_None) { + if (PyArray_FillWithScalar(result, identity) < 0) { + goto fail; + } + } + else { + /* + * For 1-D skip_first_count could be optimized to 0, but no-identity + * reductions are not super common. + * (see also comment in CopyInitialReduceValues) + */ + skip_first_count = PyArray_CopyInitialReduceValues( + result, operand, axis_flags, funcname, keepdims); + if (skip_first_count < 0) { + goto fail; + } + } + + if (!NpyIter_Reset(iter, NULL)) { + goto fail; + } + /* Start with the floating-point exception flags cleared */ npy_clear_floatstatus_barrier((char*)&iter); @@ -595,29 +373,15 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, goto fail; } - NpyIter_Deallocate(iter); - Py_DECREF(op_view); - -finish: - /* Strip out the extra 'one' dimensions in the result */ - if (out == NULL) { - if (!keepdims) { - PyArray_RemoveAxesInPlace(result, axis_flags); - } - } - else { - PyArray_ResolveWritebackIfCopy(result); /* prevent spurious warnings */ - Py_DECREF(result); + if (out != NULL) { result = out; - Py_INCREF(result); } + Py_INCREF(result); + NpyIter_Deallocate(iter); return result; fail: - PyArray_ResolveWritebackIfCopy(result); /* prevent spurious warnings */ - Py_XDECREF(result); - Py_XDECREF(op_view); if (iter != NULL) { NpyIter_Deallocate(iter); } diff --git a/numpy/core/src/umath/reduction.h b/numpy/core/src/umath/reduction.h index 0c2183ed625e..372605dba43c 100644 --- a/numpy/core/src/umath/reduction.h +++ b/numpy/core/src/umath/reduction.h @@ -128,9 +128,7 @@ typedef int (PyArray_ReduceLoopFunc)(NpyIter *iter, * of cache behavior or multithreading requirements. * keepdims : If true, leaves the reduction dimensions in the result * with size one. - * subok : If true, the result uses the subclass of operand, otherwise - * it is always a base class ndarray. - * identity : If Py_None, PyArray_InitializeReduceResult is used, otherwise + * identity : If Py_None, PyArray_CopyInitialReduceValues is used, otherwise * this value is used to initialize the result to * the reduction's unit. * loop : The loop which does the reduction. @@ -147,7 +145,6 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_CASTING casting, npy_bool *axis_flags, int reorderable, int keepdims, - int subok, PyObject *identity, PyArray_ReduceLoopFunc *loop, void *data, npy_intp buffersize, const char *funcname, diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index e473652931bf..1dbfa87e3f9a 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1903,22 +1903,34 @@ def test_reduce_noncontig_output(self): assert_equal(y_base[1,:], y_base_copy[1,:]) assert_equal(y_base[3,:], y_base_copy[3,:]) - @pytest.mark.parametrize('output_shape', - [(), (1,), (1, 1), (1, 3), (4, 3)]) + @pytest.mark.parametrize('out_shape', + [(), (1,), (3,), (1, 1), (1, 3), (4, 3)]) + @pytest.mark.parametrize('keepdims', [True, False]) @pytest.mark.parametrize('f_reduce', [np.add.reduce, np.minimum.reduce]) - def test_reduce_wrong_dimension_output(self, f_reduce, output_shape): + def test_reduce_wrong_dimension_output(self, f_reduce, keepdims, out_shape): # Test that we're not incorrectly broadcasting dimensions. # See gh-15144 (failed for np.add.reduce previously). a = np.arange(12.).reshape(4, 3) - out = np.empty(output_shape, a.dtype) - assert_raises(ValueError, f_reduce, a, axis=0, out=out) - if output_shape != (1, 3): - assert_raises(ValueError, f_reduce, a, axis=0, out=out, - keepdims=True) + out = np.empty(out_shape, a.dtype) + + correct_out = f_reduce(a, axis=0, keepdims=keepdims) + if out_shape != correct_out.shape: + with assert_raises(ValueError): + f_reduce(a, axis=0, out=out, keepdims=keepdims) else: - check = f_reduce(a, axis=0, out=out, keepdims=True) + check = f_reduce(a, axis=0, out=out, keepdims=keepdims) assert_(check is out) - assert_array_equal(check, f_reduce(a, axis=0, keepdims=True)) + assert_array_equal(check, correct_out) + + def test_reduce_output_no_subclass(self): + class MyArr(np.ndarray): + pass + + out = np.empty(()) + np.add.reduce(np.ones(5), out=out) # no subclass, all fine + out = out.view(MyArr) + assert np.add.reduce(np.ones(5), out=out) is out + assert type(np.add.reduce(out)) is MyArr def test_no_doc_string(self): # gh-9337 From c65acda08500c5c17e6f08235f8f76038763655d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 11 May 2020 17:31:19 -0500 Subject: [PATCH 0036/4003] DOC: Slightly improve error message on incorrectly shaped reductions --- numpy/core/src/multiarray/nditer_constr.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 7bc40f03aedb..c717f290f92d 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1420,13 +1420,15 @@ check_mask_for_writemasked_reduction(NpyIter *iter, int iop) */ static int npyiter_check_reduce_ok_and_set_flags( - NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags) { + NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags, + int dim) { /* If it's writeable, this means a reduction */ if (*op_itflags & NPY_OP_ITFLAG_WRITE) { if (!(flags & NPY_ITER_REDUCE_OK)) { - PyErr_SetString(PyExc_ValueError, - "output operand requires a reduction, but reduction is" - "not enabled"); + PyErr_Format(PyExc_ValueError, + "output operand requires a reduction along dimension %d, " + "but the reduction is not enabled. The dimension size of 1 " + "does not match the expected output shape.", dim); return 0; } if (!(*op_itflags & NPY_OP_ITFLAG_READ)) { @@ -1706,7 +1708,7 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf goto operand_different_than_broadcast; } if (!npyiter_check_reduce_ok_and_set_flags( - iter, flags, &op_itflags[iop])) { + iter, flags, &op_itflags[iop], i)) { return 0; } } @@ -1717,7 +1719,7 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf else { strides[iop] = 0; if (!npyiter_check_reduce_ok_and_set_flags( - iter, flags, &op_itflags[iop])) { + iter, flags, &op_itflags[iop], i)) { return 0; } } @@ -2614,7 +2616,7 @@ npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, */ if (!reduction_axis && NAD_SHAPE(axisdata) != 1) { if (!npyiter_check_reduce_ok_and_set_flags( - iter, flags, op_itflags)) { + iter, flags, op_itflags, i)) { return NULL; } } From fd89e950701206122c358bba1bf816a8bcbea844 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" Date: Wed, 13 May 2020 12:47:54 +0200 Subject: [PATCH 0037/4003] defaults for stdout and stderr should be bytes only print stdout if no OSError exception was raised --- numpy/f2py/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 007ed5d89354..6491e445fcfd 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -109,9 +109,10 @@ def compile(source, stderr=subprocess.PIPE) except OSError: # preserve historic status code used by exec_command() - cp = subprocess.CompletedProcess(c, 127, stdout='', stderr='') - if verbose: - print(cp.stdout.decode()) + cp = subprocess.CompletedProcess(c, 127, stdout=b'', stderr=b'') + else: + if verbose: + print(cp.stdout.decode()) finally: if source_fn is None: os.remove(fname) From 66365a57ff9b68a191fd232dc823cc2cf69d267f Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 7 May 2020 14:04:24 -0700 Subject: [PATCH 0038/4003] Handle TypeError in _generate_str for coefs. Add a fallback for TypeErrors that are raised when attempting to compare arbitrary elements (e.g. strings or Python complex) to 0 in _generate_str. --- numpy/polynomial/_polybase.py | 12 ++++-- numpy/polynomial/tests/test_printing.py | 49 +++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index 253aa74e81fc..7616b95c711f 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -347,10 +347,16 @@ def _generate_string(self, term_method): out += " " power = str(i + 1) # Polynomial coefficient - if coef >= 0: + try: + if coef >= 0: + next_term = f"+ {coef}" + else: + next_term = f"- {-coef}" + # The coefficient array can be an object array with elements that + # will raise a TypeError with >= 0 (e.g. strings or Python + # complex). In this case, represent the coeficient as-is. + except TypeError: next_term = f"+ {coef}" - else: - next_term = f"- {-coef}" # Polynomial term next_term += term_method(power, "x") # Length of the current line with next term added diff --git a/numpy/polynomial/tests/test_printing.py b/numpy/polynomial/tests/test_printing.py index acccb23f5fae..db71277d5dd4 100644 --- a/numpy/polynomial/tests/test_printing.py +++ b/numpy/polynomial/tests/test_printing.py @@ -1,8 +1,12 @@ import pytest -from numpy.core import arange, printoptions +from numpy.core import array, arange, printoptions import numpy.polynomial as poly from numpy.testing import assert_equal, assert_ +# For testing polynomial printing with object arrays +from fractions import Fraction +from decimal import Decimal + class TestStrUnicodeSuperSubscripts: @@ -233,11 +237,48 @@ def test_set_default_printoptions(): def test_complex_coefficients(): - p = poly.Polynomial([0+1j, 1+1j, -2+2j, 3+0j]) + """Test both numpy and built-in complex.""" + coefs = [0+1j, 1+1j, -2+2j, 3+0j] + # numpy complex + p1 = poly.Polynomial(coefs) + # Python complex + p2 = poly.Polynomial(array(coefs, dtype=object)) poly.set_default_printstyle('unicode') - assert_equal(str(p), "1j + (1+1j)·x¹ - (2-2j)·x² + (3+0j)·x³") + assert_equal(str(p1), "1j + (1+1j)·x¹ - (2-2j)·x² + (3+0j)·x³") + assert_equal(str(p2), "1j + (1+1j)·x¹ + (-2+2j)·x² + (3+0j)·x³") poly.set_default_printstyle('ascii') - assert_equal(str(p), "1j + (1+1j) x**1 - (2-2j) x**2 + (3+0j) x**3") + assert_equal(str(p1), "1j + (1+1j) x**1 - (2-2j) x**2 + (3+0j) x**3") + assert_equal(str(p2), "1j + (1+1j) x**1 + (-2+2j) x**2 + (3+0j) x**3") + + +@pytest.mark.parametrize(('coefs', 'tgt'), ( + (array([Fraction(1, 2), Fraction(3, 4)], dtype=object), ( + "1/2 + 3/4·x¹" + )), + (array([1, 2, Fraction(5, 7)], dtype=object), ( + "1 + 2·x¹ + 5/7·x²" + )), + (array([Decimal('1.00'), Decimal('2.2'), 3], dtype=object), ( + "1.00 + 2.2·x¹ + 3·x²" + )), +)) +def test_numeric_object_coefficients(coefs, tgt): + p = poly.Polynomial(coefs) + poly.set_default_printstyle('unicode') + assert_equal(str(p), tgt) + + +@pytest.mark.parametrize(('coefs', 'tgt'), ( + (array([1, 2, 'f'], dtype=object), '1 + 2·x¹ + f·x²'), + (array([1, 2, [3, 4]], dtype=object), '1 + 2·x¹ + [3, 4]·x²'), +)) +def test_nonnumeric_object_coefficients(coefs, tgt): + """ + Test coef fallback for object arrays of non-numeric coefficients. + """ + p = poly.Polynomial(coefs) + poly.set_default_printstyle('unicode') + assert_equal(str(p), tgt) class TestFormat: From 8922a6e025d53d5aaccba02b76dbef5a43afe768 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 13 May 2020 16:16:04 -0700 Subject: [PATCH 0039/4003] STY: Move comment outside of try/except. --- numpy/polynomial/_polybase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index 7616b95c711f..30887b670f79 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -347,14 +347,14 @@ def _generate_string(self, term_method): out += " " power = str(i + 1) # Polynomial coefficient + # The coefficient array can be an object array with elements that + # will raise a TypeError with >= 0 (e.g. strings or Python + # complex). In this case, represent the coeficient as-is. try: if coef >= 0: next_term = f"+ {coef}" else: next_term = f"- {-coef}" - # The coefficient array can be an object array with elements that - # will raise a TypeError with >= 0 (e.g. strings or Python - # complex). In this case, represent the coeficient as-is. except TypeError: next_term = f"+ {coef}" # Polynomial term From bf14c1ce748f46df7daf5923cfb79ff905e233e8 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 13 May 2020 22:10:54 -0500 Subject: [PATCH 0040/4003] MAINT: Unify cached (C-level static) imports The main idea is that once we unify them, we can think about actually being able to also clean them up. This is far away from having no global state, but at this time I do not see that as a seriousl goal personally. But it could be possible to reload the module safely in a single interpreter maybe. The commit adds the cache import for the complex casts, since it doubles performance for small casts when the warning is not suppressed and it unifies the handling. --- numpy/core/src/common/npy_import.h | 2 +- numpy/core/src/multiarray/common.h | 9 ++---- numpy/core/src/multiarray/dtype_transfer.c | 10 +++---- numpy/core/src/multiarray/methods.c | 33 ++-------------------- numpy/core/src/umath/scalarmath.c.src | 9 ++---- 5 files changed, 13 insertions(+), 50 deletions(-) diff --git a/numpy/core/src/common/npy_import.h b/numpy/core/src/common/npy_import.h index 221e1e645a47..f485514d1cd1 100644 --- a/numpy/core/src/common/npy_import.h +++ b/numpy/core/src/common/npy_import.h @@ -19,7 +19,7 @@ NPY_INLINE static void npy_cache_import(const char *module, const char *attr, PyObject **cache) { - if (*cache == NULL) { + if (NPY_UNLIKELY(*cache == NULL)) { PyObject *mod = PyImport_ImportModule(module); if (mod != NULL) { diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 4913eb202f22..78a15a63ccf0 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -5,6 +5,7 @@ #include #include #include +#include "npy_import.h" #define error_converting(x) (((x) == -1) && PyErr_Occurred()) @@ -148,13 +149,9 @@ check_and_adjust_axis_msg(int *axis, int ndim, PyObject *msg_prefix) static PyObject *AxisError_cls = NULL; PyObject *exc; + npy_cache_import("numpy.core._exceptions", "AxisError", &AxisError_cls); if (AxisError_cls == NULL) { - PyObject *mod = PyImport_ImportModule("numpy.core._exceptions"); - - if (mod != NULL) { - AxisError_cls = PyObject_GetAttrString(mod, "AxisError"); - Py_DECREF(mod); - } + return -1; } /* Invoke the AxisError constructor */ diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index ecaa680ecf8a..a26426d41549 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -696,17 +696,15 @@ get_nbo_cast_numeric_transfer_function(int aligned, if (PyTypeNum_ISCOMPLEX(src_type_num) && !PyTypeNum_ISCOMPLEX(dst_type_num) && !PyTypeNum_ISBOOL(dst_type_num)) { - PyObject *cls = NULL, *obj = NULL; + static PyObject *cls = NULL; int ret; - obj = PyImport_ImportModule("numpy.core"); - if (obj) { - cls = PyObject_GetAttrString(obj, "ComplexWarning"); - Py_DECREF(obj); + npy_cache_import("numpy.core", "ComplexWarning", &cls); + if (cls == NULL) { + return NPY_FAIL; } ret = PyErr_WarnEx(cls, "Casting complex values to real discards " "the imaginary part", 1); - Py_XDECREF(cls); if (ret < 0) { return NPY_FAIL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index e2026ec1c3ab..8a659fa6982d 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -54,33 +54,6 @@ NpyArg_ParseKeywords(PyObject *keys, const char *format, char **kwlist, ...) return ret; } -static PyObject * -get_forwarding_ndarray_method(const char *name) -{ - PyObject *module_methods, *callable; - - /* Get a reference to the function we're calling */ - module_methods = PyImport_ImportModule("numpy.core._methods"); - if (module_methods == NULL) { - return NULL; - } - callable = _PyDict_GetItemStringWithError(PyModule_GetDict(module_methods), name); - if (callable == NULL && PyErr_Occurred()) { - Py_DECREF(module_methods); - return NULL; - } - if (callable == NULL) { - Py_DECREF(module_methods); - PyErr_Format(PyExc_RuntimeError, - "NumPy internal error: could not find function " - "numpy.core._methods.%s", name); - } - else { - Py_INCREF(callable); - } - Py_DECREF(module_methods); - return callable; -} /* * Forwards an ndarray method to a the Python function @@ -121,11 +94,9 @@ forward_ndarray_method(PyArrayObject *self, PyObject *args, PyObject *kwds, */ #define NPY_FORWARD_NDARRAY_METHOD(name) \ static PyObject *callable = NULL; \ + npy_cache_import("numpy.core._methods", name, &callable); \ if (callable == NULL) { \ - callable = get_forwarding_ndarray_method(name); \ - if (callable == NULL) { \ - return NULL; \ - } \ + return NULL; \ } \ return forward_ndarray_method(self, args, kwds, callable) diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index bb2915e09128..90cc7a513418 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -16,6 +16,7 @@ #include "numpy/ufuncobject.h" #include "numpy/arrayscalars.h" +#include "npy_import.h" #include "npy_pycompat.h" #include "numpy/halffloat.h" @@ -1339,13 +1340,9 @@ static int emit_complexwarning(void) { static PyObject *cls = NULL; + npy_cache_import("numpy.core", "ComplexWarning", &cls); if (cls == NULL) { - PyObject *mod; - mod = PyImport_ImportModule("numpy.core"); - assert(mod != NULL); - cls = PyObject_GetAttrString(mod, "ComplexWarning"); - assert(cls != NULL); - Py_DECREF(mod); + return -1; } return PyErr_WarnEx(cls, "Casting complex values to real discards the imaginary part", 1); From 4dc9ff62a78004d6debd0953d70cd39864b9d6c3 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 15 May 2020 11:46:09 -0700 Subject: [PATCH 0041/4003] DOC: Update poly class refguide printing. Update routines.polynomials.classes doc in the refguide to reflect changes to polynomial printing. Add additional information to the document about the various ways that the string representation of polynomial expressions can be controlled via formatting. --- .../routines.polynomials.classes.rst | 21 ++++++++++++++++--- .../routines.polynomials.package.rst | 8 +++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/routines.polynomials.classes.rst b/doc/source/reference/routines.polynomials.classes.rst index 71e635866679..10331e9c17a4 100644 --- a/doc/source/reference/routines.polynomials.classes.rst +++ b/doc/source/reference/routines.polynomials.classes.rst @@ -65,11 +65,26 @@ window:: >>> p.window array([-1., 1.]) -Printing a polynomial yields a shorter form without the domain -and window:: +Printing a polynomial yields the polynomial expression in a more familiar +format:: >>> print(p) - poly([1. 2. 3.]) + 1.0 + 2.0·x¹ + 3.0·x² + +Note that the string representation of polynomials uses Unicode characters +by default (except on Windows) to express powers and subscripts. An ASCII-based +representation is also available (default on Windows). The polynomial string +format can be toggled at the package-level with the +`~numpy.polynomial.set_default_printstyle` function:: + + >>> numpy.polynomial.set_default_printstyle('ascii') + >>> print(p) + 1.0 + 2.0 x**1 + 3.0 x**2 + +or controlled for individual polynomial instances with string formatting:: + + >>> print(f"{p:unicode}") + 1.0 + 2.0·x¹ + 3.0·x² We will deal with the domain and window when we get to fitting, for the moment we ignore them and run through the basic algebraic and arithmetic operations. diff --git a/doc/source/reference/routines.polynomials.package.rst b/doc/source/reference/routines.polynomials.package.rst index ca1217f80050..1bc528c59495 100644 --- a/doc/source/reference/routines.polynomials.package.rst +++ b/doc/source/reference/routines.polynomials.package.rst @@ -4,3 +4,11 @@ :no-members: :no-inherited-members: :no-special-members: + +Configuration +------------- + +.. autosummary:: + :toctree: generated/ + + numpy.polynomial.set_default_printstyle From 00e76889d0f35de7415bf6967237351972cb44cd Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Fri, 15 May 2020 20:18:52 -0400 Subject: [PATCH 0042/4003] DOC: Update the f2py section of the "Using Python as Glue" page. * Update the output shown for the docstrings generated by f2py. --- doc/source/user/c-info.python-as-glue.rst | 37 ++++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/doc/source/user/c-info.python-as-glue.rst b/doc/source/user/c-info.python-as-glue.rst index 201bd8417ac7..6904a2fec89c 100644 --- a/doc/source/user/c-info.python-as-glue.rst +++ b/doc/source/user/c-info.python-as-glue.rst @@ -171,14 +171,16 @@ information about how the module method may be called:: >>> import add >>> print(add.zadd.__doc__) - zadd - Function signature: - zadd(a,b,c,n) - Required arguments: - a : input rank-1 array('D') with bounds (*) - b : input rank-1 array('D') with bounds (*) - c : input rank-1 array('D') with bounds (*) - n : input int + zadd(a,b,c,n) + Wrapper for ``zadd``. + + Parameters + ---------- + a : input rank-1 array('D') with bounds (*) + b : input rank-1 array('D') with bounds (*) + c : input rank-1 array('D') with bounds (*) + n : input int Improving the basic interface ----------------------------- @@ -249,18 +251,23 @@ The new interface has docstring:: >>> import add >>> print(add.zadd.__doc__) - zadd - Function signature: - c = zadd(a,b) - Required arguments: - a : input rank-1 array('D') with bounds (n) - b : input rank-1 array('D') with bounds (n) - Return objects: - c : rank-1 array('D') with bounds (n) + c = zadd(a,b) + + Wrapper for ``zadd``. + + Parameters + ---------- + a : input rank-1 array('D') with bounds (n) + b : input rank-1 array('D') with bounds (n) + + Returns + ------- + c : rank-1 array('D') with bounds (n) Now, the function can be called in a much more robust way:: >>> add.zadd([1,2,3],[4,5,6]) - array([ 5.+0.j, 7.+0.j, 9.+0.j]) + array([5.+0.j, 7.+0.j, 9.+0.j]) Notice the automatic conversion to the correct format that occurred. From c2cbf13ac4df693cec70f6949efdc386c791d1f9 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 17 May 2020 15:05:19 +0100 Subject: [PATCH 0043/4003] MAINT: Remove a pointless if The `add` ufunc is happy to handle `out=None` by itself --- numpy/lib/function_base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index dea01d12d906..12ba3b1c6811 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3975,10 +3975,7 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, x1 = x1.squeeze(0) x2 = x2.squeeze(0) - if out is not None: - r = add(x1, x2, out=out) - else: - r = add(x1, x2) + r = add(x1, x2, out=out) if np.any(n): if zerod: From 49a84f6fd548c0d01d5dbd6ee286cdaa49fd6209 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 17 May 2020 15:14:58 +0100 Subject: [PATCH 0044/4003] MAINT: Avoid moving axes around multiple times It's easier to move the relevant axis to position 0 in `ap` first than it is to move it for every relevant object simultaneously. --- numpy/lib/function_base.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 12ba3b1c6811..2d1adc362f61 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3943,34 +3943,24 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, if np.issubdtype(a.dtype, np.inexact): indices_above = concatenate((indices_above, [-1])) - weights_above = indices - indices_below - weights_below = 1 - weights_above - - weights_shape = [1, ] * ap.ndim - weights_shape[axis] = len(indices) - weights_below.shape = weights_shape - weights_above.shape = weights_shape - ap.partition(concatenate((indices_below, indices_above)), axis=axis) # ensure axis with q-th is first ap = np.moveaxis(ap, axis, 0) - weights_below = np.moveaxis(weights_below, axis, 0) - weights_above = np.moveaxis(weights_above, axis, 0) axis = 0 + weights_shape = [1] * ap.ndim + weights_shape[axis] = len(indices) + weights_above = (indices - indices_below).reshape(weights_shape) + # Check if the array contains any nan's if np.issubdtype(a.dtype, np.inexact): indices_above = indices_above[:-1] n = np.isnan(ap[-1:, ...]) - x1 = take(ap, indices_below, axis=axis) * weights_below + x1 = take(ap, indices_below, axis=axis) * (1 - weights_above) x2 = take(ap, indices_above, axis=axis) * weights_above - # ensure axis with q-th is first - x1 = np.moveaxis(x1, axis, 0) - x2 = np.moveaxis(x2, axis, 0) - if zerod: x1 = x1.squeeze(0) x2 = x2.squeeze(0) From b578312d381ead845e403ae09a3fc7f7ccdaed92 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 17 May 2020 07:59:23 -0600 Subject: [PATCH 0045/4003] REL: Update master after 1.19.x branch. - Delete release note fragments from 1.19.0 (towncrier) - Update 1.19.x release note (towncrier) - Create 1.20.0-notes.rst for master development - Update C-API versions for 1.20.x - Update setup.py for 1.20.0 --- .../upcoming_changes/13421.improvement.rst | 8 - .../upcoming_changes/14924.compatibility.rst | 7 - .../upcoming_changes/14933.compatibility.rst | 10 - .../upcoming_changes/14942.compatibility.rst | 6 - .../upcoming_changes/14981.compatibility.rst | 10 - .../upcoming_changes/14995.compatibility.rst | 10 - doc/release/upcoming_changes/15118.change.rst | 7 - .../upcoming_changes/15119.deprecation.rst | 8 - .../upcoming_changes/15217.deprecation.rst | 13 - .../upcoming_changes/15218.improvement.rst | 6 - .../upcoming_changes/15229.compatibility.rst | 7 - .../upcoming_changes/15233.highlight.rst | 4 - doc/release/upcoming_changes/15251.c_api.rst | 10 - .../upcoming_changes/15255.compatibility.rst | 12 - doc/release/upcoming_changes/15355.c_api.rst | 7 - .../upcoming_changes/15385.new_feature.rst | 5 - .../upcoming_changes/15427.deprecation.rst | 12 - doc/release/upcoming_changes/15463.change.rst | 12 - .../upcoming_changes/15534.deprecation.rst | 11 - .../upcoming_changes/15648.improvement.rst | 7 - .../upcoming_changes/15685.new_feature.rst | 9 - .../upcoming_changes/15715.new_feature.rst | 5 - .../upcoming_changes/15769.improvement.rst | 15 - .../upcoming_changes/15773.compatibility.rst | 10 - .../upcoming_changes/15802.expired.rst | 9 - .../upcoming_changes/15804.expired.rst | 8 - .../upcoming_changes/15805.expired.rst | 6 - .../upcoming_changes/15815.expired.rst | 6 - .../upcoming_changes/15840.compatibility.rst | 7 - .../upcoming_changes/15840.deprecation.rst | 6 - .../upcoming_changes/15867.deprecation.rst | 6 - .../upcoming_changes/15870.new_feature.rst | 5 - doc/release/upcoming_changes/15872.change.rst | 6 - .../upcoming_changes/15882.compatibility.rst | 5 - .../upcoming_changes/16068.compatibility.rst | 11 - .../upcoming_changes/16080.improvement.rst | 4 - .../upcoming_changes/16102.improvement.rst | 4 - .../upcoming_changes/16128.new_feature.rst | 6 - doc/release/upcoming_changes/16153.change.rst | 10 - .../upcoming_changes/8255.new_feature.rst | 5 - doc/source/release.rst | 1 + doc/source/release/1.19.0-notes.rst | 469 ++++++++++++++++++ doc/source/release/1.20.0-notes.rst | 6 + numpy/core/code_generators/cversions.txt | 1 + numpy/core/include/numpy/numpyconfig.h | 1 + setup.py | 2 +- 46 files changed, 479 insertions(+), 316 deletions(-) delete mode 100644 doc/release/upcoming_changes/13421.improvement.rst delete mode 100644 doc/release/upcoming_changes/14924.compatibility.rst delete mode 100644 doc/release/upcoming_changes/14933.compatibility.rst delete mode 100644 doc/release/upcoming_changes/14942.compatibility.rst delete mode 100644 doc/release/upcoming_changes/14981.compatibility.rst delete mode 100644 doc/release/upcoming_changes/14995.compatibility.rst delete mode 100644 doc/release/upcoming_changes/15118.change.rst delete mode 100644 doc/release/upcoming_changes/15119.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15217.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15218.improvement.rst delete mode 100644 doc/release/upcoming_changes/15229.compatibility.rst delete mode 100644 doc/release/upcoming_changes/15233.highlight.rst delete mode 100644 doc/release/upcoming_changes/15251.c_api.rst delete mode 100644 doc/release/upcoming_changes/15255.compatibility.rst delete mode 100644 doc/release/upcoming_changes/15355.c_api.rst delete mode 100644 doc/release/upcoming_changes/15385.new_feature.rst delete mode 100644 doc/release/upcoming_changes/15427.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15463.change.rst delete mode 100644 doc/release/upcoming_changes/15534.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15648.improvement.rst delete mode 100644 doc/release/upcoming_changes/15685.new_feature.rst delete mode 100644 doc/release/upcoming_changes/15715.new_feature.rst delete mode 100644 doc/release/upcoming_changes/15769.improvement.rst delete mode 100644 doc/release/upcoming_changes/15773.compatibility.rst delete mode 100644 doc/release/upcoming_changes/15802.expired.rst delete mode 100644 doc/release/upcoming_changes/15804.expired.rst delete mode 100644 doc/release/upcoming_changes/15805.expired.rst delete mode 100644 doc/release/upcoming_changes/15815.expired.rst delete mode 100644 doc/release/upcoming_changes/15840.compatibility.rst delete mode 100644 doc/release/upcoming_changes/15840.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15867.deprecation.rst delete mode 100644 doc/release/upcoming_changes/15870.new_feature.rst delete mode 100644 doc/release/upcoming_changes/15872.change.rst delete mode 100644 doc/release/upcoming_changes/15882.compatibility.rst delete mode 100644 doc/release/upcoming_changes/16068.compatibility.rst delete mode 100644 doc/release/upcoming_changes/16080.improvement.rst delete mode 100644 doc/release/upcoming_changes/16102.improvement.rst delete mode 100644 doc/release/upcoming_changes/16128.new_feature.rst delete mode 100644 doc/release/upcoming_changes/16153.change.rst delete mode 100644 doc/release/upcoming_changes/8255.new_feature.rst create mode 100644 doc/source/release/1.20.0-notes.rst diff --git a/doc/release/upcoming_changes/13421.improvement.rst b/doc/release/upcoming_changes/13421.improvement.rst deleted file mode 100644 index 6d4573aa38bc..000000000000 --- a/doc/release/upcoming_changes/13421.improvement.rst +++ /dev/null @@ -1,8 +0,0 @@ -Improve detection of CPU features -================================= - -Replace ``npy_cpu_supports`` which was a gcc-specific mechanism to test support -of avx with more general functions ``npy_cpu_init`` and ``npy_cpu_have``, and -expose the results via a ``NPY_CPU_HAVE`` c-macro as well as a python-level -``__cpu_features__`` dictionary. - diff --git a/doc/release/upcoming_changes/14924.compatibility.rst b/doc/release/upcoming_changes/14924.compatibility.rst deleted file mode 100644 index 8b42437fd502..000000000000 --- a/doc/release/upcoming_changes/14924.compatibility.rst +++ /dev/null @@ -1,7 +0,0 @@ -Changed random variate stream from `numpy.random.Generator.dirichlet` ---------------------------------------------------------------------- -A bug in the generation of random variates for the Dirichlet distribution -with small `alpha` values was fixed by using a different algorithm when -``max(alpha) < 0.1``. Because of the change, the stream of variates -generated by `dirichlet` in this case will be different from previous -releases. diff --git a/doc/release/upcoming_changes/14933.compatibility.rst b/doc/release/upcoming_changes/14933.compatibility.rst deleted file mode 100644 index 1b5f1b113ddc..000000000000 --- a/doc/release/upcoming_changes/14933.compatibility.rst +++ /dev/null @@ -1,10 +0,0 @@ -Scalar promotion in ``PyArray_ConvertToCommonType`` ---------------------------------------------------- - -The promotion of mixed scalars and arrays in ``PyArray_ConvertToCommonType`` -has been changed to adhere to those used by ``np.result_type``. -This means that input such as ``(1000, np.array([1], dtype=np.uint8)))`` -will now return ``uint16`` dtypes. In most cases the behaviour is unchanged. -Note that the use of this C-API function is generally discouraged. -This also fixes ``np.choose`` to behave the same way as the rest of NumPy -in this respect. diff --git a/doc/release/upcoming_changes/14942.compatibility.rst b/doc/release/upcoming_changes/14942.compatibility.rst deleted file mode 100644 index 461758c95a68..000000000000 --- a/doc/release/upcoming_changes/14942.compatibility.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fasttake and fastputmask slots are deprecated and NULL'ed ---------------------------------------------------------- -The fasttake and fastputmask slots are now never used and -must always be set to NULL. This will result in no change in behaviour. -However, if a user dtype should set one of these a DeprecationWarning -will be given. diff --git a/doc/release/upcoming_changes/14981.compatibility.rst b/doc/release/upcoming_changes/14981.compatibility.rst deleted file mode 100644 index 90cf866f21b4..000000000000 --- a/doc/release/upcoming_changes/14981.compatibility.rst +++ /dev/null @@ -1,10 +0,0 @@ -`np.ediff1d` casting behaviour with ``to_end`` and ``to_begin`` ---------------------------------------------------------------- - -`np.ediff1d` now uses the ``"same_kind"`` casting rule for -its additional ``to_end`` and ``to_begin`` arguments. This -ensures type safety except when the input array has a smaller -integer type than ``to_begin`` or ``to_end``. -In rare cases, the behaviour will be more strict than it was -previously in 1.16 and 1.17. This is necessary to solve issues -with floating point NaN. diff --git a/doc/release/upcoming_changes/14995.compatibility.rst b/doc/release/upcoming_changes/14995.compatibility.rst deleted file mode 100644 index 140e66486164..000000000000 --- a/doc/release/upcoming_changes/14995.compatibility.rst +++ /dev/null @@ -1,10 +0,0 @@ -Converting of empty array-like objects to NumPy arrays ------------------------------------------------------- -Objects with ``len(obj) == 0`` which implement an "array-like" interface, -meaning an object implementing ``obj.__array__()``, -``obj.__array_interface__``, ``obj.__array_struct__``, or the python -buffer interface and which are also sequences (i.e. Pandas objects) -will now always retain there shape correctly when converted to an array. -If such an object has a shape of ``(0, 1)`` previously, it could -be converted into an array of shape ``(0,)`` (losing all dimensions -after the first 0). diff --git a/doc/release/upcoming_changes/15118.change.rst b/doc/release/upcoming_changes/15118.change.rst deleted file mode 100644 index f14beebbe9e4..000000000000 --- a/doc/release/upcoming_changes/15118.change.rst +++ /dev/null @@ -1,7 +0,0 @@ -Remove handling of extra argument to ``__array__`` --------------------------------------------------- -A code path and test have been in the code since NumPy 0.4 for a two-argument -variant of ``__array__(dtype=None, context=None)``. It was activated when -calling ``ufunc(op)`` or ``ufunc.reduce(op)`` if ``op.__array__`` existed. -However that variant is not documented, and it is not clear what the intention -was for its use. It has been removed. diff --git a/doc/release/upcoming_changes/15119.deprecation.rst b/doc/release/upcoming_changes/15119.deprecation.rst deleted file mode 100644 index d18e440fe0e7..000000000000 --- a/doc/release/upcoming_changes/15119.deprecation.rst +++ /dev/null @@ -1,8 +0,0 @@ - -Deprecate automatic ``dtype=object`` for ragged input ------------------------------------------------------ -Calling ``np.array([[1, [1, 2, 3]])`` will issue a ``DeprecationWarning`` as -per `NEP 34`_. Users should explicitly use ``dtype=object`` to avoid the -warning. - -.. _`NEP 34`: https://numpy.org/neps/nep-0034.html diff --git a/doc/release/upcoming_changes/15217.deprecation.rst b/doc/release/upcoming_changes/15217.deprecation.rst deleted file mode 100644 index d49de20b512f..000000000000 --- a/doc/release/upcoming_changes/15217.deprecation.rst +++ /dev/null @@ -1,13 +0,0 @@ -Passing ``shape=0`` to factory functions in ``numpy.rec`` is deprecated ------------------------------------------------------------------------ - -``0`` is treated as a special case and is aliased to ``None`` in the functions: - -* `numpy.core.records.fromarrays` -* `numpy.core.records.fromrecords` -* `numpy.core.records.fromstring` -* `numpy.core.records.fromfile` - -In future, ``0`` will not be special cased, and will be treated as an array -length like any other integer. - diff --git a/doc/release/upcoming_changes/15218.improvement.rst b/doc/release/upcoming_changes/15218.improvement.rst deleted file mode 100644 index ccbbbd66f516..000000000000 --- a/doc/release/upcoming_changes/15218.improvement.rst +++ /dev/null @@ -1,6 +0,0 @@ -Use 64-bit integer size on 64-bit platforms in fallback lapack_lite -------------------------------------------------------------------- - -Use 64-bit integer size on 64-bit platforms in the fallback LAPACK library, -which is used when the system has no LAPACK installed, allowing it to deal with -linear algebra for large arrays. diff --git a/doc/release/upcoming_changes/15229.compatibility.rst b/doc/release/upcoming_changes/15229.compatibility.rst deleted file mode 100644 index 404f7774fd64..000000000000 --- a/doc/release/upcoming_changes/15229.compatibility.rst +++ /dev/null @@ -1,7 +0,0 @@ -Removed ``multiarray.int_asbuffer`` ------------------------------------ - -As part of the continued removal of Python 2 compatibility, -``multiarray.int_asbuffer`` was removed. On Python 3, it threw a -``NotImplementedError`` and was unused internally. It is expected that there -are no downstream use cases for this method with Python 3. diff --git a/doc/release/upcoming_changes/15233.highlight.rst b/doc/release/upcoming_changes/15233.highlight.rst deleted file mode 100644 index df96ee87198b..000000000000 --- a/doc/release/upcoming_changes/15233.highlight.rst +++ /dev/null @@ -1,4 +0,0 @@ -* Code compatibility with Python versions < 3.5 (including Python 2) was - dropped from both the python and C code. The shims in numpy.compat will - remain to support third-party packages, but they may be deprecated in a - future release. diff --git a/doc/release/upcoming_changes/15251.c_api.rst b/doc/release/upcoming_changes/15251.c_api.rst deleted file mode 100644 index f391c904b2c2..000000000000 --- a/doc/release/upcoming_changes/15251.c_api.rst +++ /dev/null @@ -1,10 +0,0 @@ -Better support for ``const`` dimensions in API functions --------------------------------------------------------- -The following functions now accept a constant array of ``npy_intp``: - -* `PyArray_BroadcastToShape` -* `PyArray_IntTupleFromIntp` -* `PyArray_OverflowMultiplyList` - -Previously the caller would have to cast away the const-ness to call these -functions. diff --git a/doc/release/upcoming_changes/15255.compatibility.rst b/doc/release/upcoming_changes/15255.compatibility.rst deleted file mode 100644 index e360eeeb3014..000000000000 --- a/doc/release/upcoming_changes/15255.compatibility.rst +++ /dev/null @@ -1,12 +0,0 @@ -``numpy.distutils.compat`` has been removed -------------------------------------------- -This module contained only the function ``get_exception()``, which was used as:: - - try: - ... - except Exception: - e = get_exception() - -Its purpose was to handle the change in syntax introduced in Python 2.6, from -``except Exception, e:`` to ``except Exception as e:``, meaning it was only -necessary for codebases supporting Python 2.5 and older. diff --git a/doc/release/upcoming_changes/15355.c_api.rst b/doc/release/upcoming_changes/15355.c_api.rst deleted file mode 100644 index ffc1972cf86a..000000000000 --- a/doc/release/upcoming_changes/15355.c_api.rst +++ /dev/null @@ -1,7 +0,0 @@ -Const qualify UFunc inner loops -------------------------------- -``UFuncGenericFunction`` now expects pointers to const ``dimension`` and -``strides`` as arguments. This means inner loops may no longer modify -either ``dimension`` or ``strides``. This change leads to an -``incompatible-pointer-types`` warning forcing users to either ignore -the compiler warnings or to const qualify their own loop signatures. diff --git a/doc/release/upcoming_changes/15385.new_feature.rst b/doc/release/upcoming_changes/15385.new_feature.rst deleted file mode 100644 index 24e9b793c4af..000000000000 --- a/doc/release/upcoming_changes/15385.new_feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -``np.str_`` scalars now support the buffer protocol ---------------------------------------------------- -``np.str_`` arrays are always stored as UCS4, so the corresponding scalars -now expose this through the buffer interface, meaning -``memoryview(np.str_('test'))`` now works. diff --git a/doc/release/upcoming_changes/15427.deprecation.rst b/doc/release/upcoming_changes/15427.deprecation.rst deleted file mode 100644 index 0100406b3032..000000000000 --- a/doc/release/upcoming_changes/15427.deprecation.rst +++ /dev/null @@ -1,12 +0,0 @@ -Deprecation of probably unused C-API functions ----------------------------------------------- -The following C-API functions are probably unused and have been -deprecated: - -* ``PyArray_GetArrayParamsFromObject`` -* ``PyUFunc_GenericFunction`` -* ``PyUFunc_SetUsesArraysAsData`` - -In most cases ``PyArray_GetArrayParamsFromObject`` should be replaced -by converting to an array, while ``PyUFunc_GenericFunction`` can be -replaced with ``PyObject_Call`` (see documentation for details). diff --git a/doc/release/upcoming_changes/15463.change.rst b/doc/release/upcoming_changes/15463.change.rst deleted file mode 100644 index 25befbcc7086..000000000000 --- a/doc/release/upcoming_changes/15463.change.rst +++ /dev/null @@ -1,12 +0,0 @@ -``numpy.random._bit_generator`` moved to ``numpy.random.bit_generator`` ------------------------------------------------------------------------ - -In order to expose `numpy.random.BitGenerator` and `numpy.random.SeedSequence` -to cython, the ``_bitgenerator`` module is now public as -`numpy.random.bit_generator` - -Cython access to the random distributions is provided via a `pxd` file ----------------------------------------------------------------------- - -``c_distributions.pxd`` provides access to the c functions behind many of the -random distributions from Cython, making it convenient to use and extend them. diff --git a/doc/release/upcoming_changes/15534.deprecation.rst b/doc/release/upcoming_changes/15534.deprecation.rst deleted file mode 100644 index 243e224ba6b7..000000000000 --- a/doc/release/upcoming_changes/15534.deprecation.rst +++ /dev/null @@ -1,11 +0,0 @@ -Converting certain types to dtypes is Deprecated ------------------------------------------------- -The super classes of scalar types, such as ``np.integer``, ``np.generic``, -or ``np.inexact`` will now give a deprecation warning when converted -to a dtype (or used in a dtype keyword argument). -The reason for this is that `np.integer` is converted to ``np.int_``, -while it would be expected to represent *any* integer (e.g. also -``int8``, ``int16``, etc. -For example, ``dtype=np.floating`` is currently identical to -``dtype=np.float64``, even though also ``np.float32`` is a subclass of -``np.floating``. diff --git a/doc/release/upcoming_changes/15648.improvement.rst b/doc/release/upcoming_changes/15648.improvement.rst deleted file mode 100644 index 2dbfa9f9981f..000000000000 --- a/doc/release/upcoming_changes/15648.improvement.rst +++ /dev/null @@ -1,7 +0,0 @@ -Use AVX512 intrinsic to implement ``np.exp`` when input is ``np.float64`` --------------------------------------------------------------------------- -Use AVX512 intrinsic to implement ``np.exp`` when input is ``np.float64``, -which can improve the performance of ``np.exp`` with ``np.float64`` input 5-7x -faster than before. The _multiarray_umath.so module has grown about 63 KB on -linux64. - diff --git a/doc/release/upcoming_changes/15685.new_feature.rst b/doc/release/upcoming_changes/15685.new_feature.rst deleted file mode 100644 index c4ed04e93dba..000000000000 --- a/doc/release/upcoming_changes/15685.new_feature.rst +++ /dev/null @@ -1,9 +0,0 @@ -``subok`` option for `numpy.copy` ---------------------------------- -A new kwarg, ``subok``, was added to `numpy.copy` to allow users to toggle the -behavior of `numpy.copy` with respect to array subclasses. The default value -is ``False`` which is consistent with the behavior of `numpy.copy` for -previous numpy versions. To create a copy that preserves an array subclass with -`numpy.copy`, call ``np.copy(arr, subok=True)``. This addition better documents -that the default behavior of `numpy.copy` differs from the -`numpy.ndarray.copy` method which respects array subclasses by default. diff --git a/doc/release/upcoming_changes/15715.new_feature.rst b/doc/release/upcoming_changes/15715.new_feature.rst deleted file mode 100644 index a568aee643b3..000000000000 --- a/doc/release/upcoming_changes/15715.new_feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -`numpy.linalg.multi_dot` now accepts an ``out`` argument --------------------------------------------------------- - -``out`` can be used to avoid creating unnecessary copies of the final product -computed by `numpy.linalg.multidot`. diff --git a/doc/release/upcoming_changes/15769.improvement.rst b/doc/release/upcoming_changes/15769.improvement.rst deleted file mode 100644 index 3f70058f6729..000000000000 --- a/doc/release/upcoming_changes/15769.improvement.rst +++ /dev/null @@ -1,15 +0,0 @@ -Ability to disable madvise hugepages ------------------------------------- - -On Linux NumPy has previously added support for madavise -hugepages which can improve performance for very large arrays. -Unfortunately, on older Kernel versions this led to peformance -regressions, thus by default the support has been disabled on -kernels before version 4.6. To override the default, you can -use the environment variable:: - - NUMPY_MADVISE_HUGEPAGE=0 - -or set it to 1 to force enabling support. Note that this only makes -a difference if the operating system is set up to use madvise -transparent hugepage. diff --git a/doc/release/upcoming_changes/15773.compatibility.rst b/doc/release/upcoming_changes/15773.compatibility.rst deleted file mode 100644 index bc9d47153579..000000000000 --- a/doc/release/upcoming_changes/15773.compatibility.rst +++ /dev/null @@ -1,10 +0,0 @@ -``issubdtype`` no longer interprets ``float`` as ``np.floating`` ----------------------------------------------------------------- - -`numpy.issubdtype` had a FutureWarning since NumPy 1.14 which -has expired now. This means that certain input where the second -argument was neither a datatype nor a NumPy scalar type -(such as a string or a python type like ``int`` or ``float``) -will now be consistent with passing in ``np.dtype(arg2).type``. -This makes the result consistent with expectations and leads to -a false result in some cases which previously returned true. diff --git a/doc/release/upcoming_changes/15802.expired.rst b/doc/release/upcoming_changes/15802.expired.rst deleted file mode 100644 index 1a1b373a77f9..000000000000 --- a/doc/release/upcoming_changes/15802.expired.rst +++ /dev/null @@ -1,9 +0,0 @@ -`numpy.insert` and `numpy.delete` can no longer be passed an axis on 0d arrays ------------------------------------------------------------------------------- -This concludes a deprecation from 1.9, where when an ``axis`` argument was -passed to a call to `~numpy.insert` and `~numpy.delete` on a 0d array, the -``axis`` and ``obj`` argument and indices would be completely ignored. -In these cases, ``insert(arr, "nonsense", 42, axis=0)`` would actually overwrite the -entire array, while ``delete(arr, "nonsense", axis=0)`` would be ``arr.copy()`` - -Now passing ``axis`` on a 0d array raises `~numpy.AxisError`. diff --git a/doc/release/upcoming_changes/15804.expired.rst b/doc/release/upcoming_changes/15804.expired.rst deleted file mode 100644 index e110e1eadf88..000000000000 --- a/doc/release/upcoming_changes/15804.expired.rst +++ /dev/null @@ -1,8 +0,0 @@ -`numpy.delete` no longer ignores out-of-bounds indices ------------------------------------------------------- -This concludes deprecations from 1.8 and 1.9, where ``np.delete`` would ignore -both negative and out-of-bounds items in a sequence of indices. This was at -odds with its behavior when passed a single index. - -Now out-of-bounds items throw ``IndexError``, and negative items index from the -end. diff --git a/doc/release/upcoming_changes/15805.expired.rst b/doc/release/upcoming_changes/15805.expired.rst deleted file mode 100644 index d317e98b8597..000000000000 --- a/doc/release/upcoming_changes/15805.expired.rst +++ /dev/null @@ -1,6 +0,0 @@ -`numpy.insert` and `numpy.delete` no longer accept non-integral indices ------------------------------------------------------------------------ -This concludes a deprecation from 1.9, where sequences of non-integers indices -were allowed and cast to integers. Now passing sequences of non-integral -indices raises ``IndexError``, just like it does when passing a single -non-integral scalar. diff --git a/doc/release/upcoming_changes/15815.expired.rst b/doc/release/upcoming_changes/15815.expired.rst deleted file mode 100644 index b853cc9e5a7d..000000000000 --- a/doc/release/upcoming_changes/15815.expired.rst +++ /dev/null @@ -1,6 +0,0 @@ -`numpy.delete` no longer casts boolean indices to integers ----------------------------------------------------------- -This concludes a deprecation from 1.8, where ``np.delete`` would cast boolean -arrays and scalars passed as an index argument into integer indices. The -behavior now is to treat boolean arrays as a mask, and to raise an error -on boolean scalars. diff --git a/doc/release/upcoming_changes/15840.compatibility.rst b/doc/release/upcoming_changes/15840.compatibility.rst deleted file mode 100644 index 2709f133da7f..000000000000 --- a/doc/release/upcoming_changes/15840.compatibility.rst +++ /dev/null @@ -1,7 +0,0 @@ -Change output of ``round`` on scalars to be consistent with Python ------------------------------------------------------------------- - -Output of the ``__round__`` dunder method and consequently the Python -built-in `round` has been changed to be a Python `int` to be consistent -with calling it on Python ``float`` objects when called with no arguments. -Previously, it would return a scalar of the `np.dtype` that was passed in. diff --git a/doc/release/upcoming_changes/15840.deprecation.rst b/doc/release/upcoming_changes/15840.deprecation.rst deleted file mode 100644 index cfcfecdb1282..000000000000 --- a/doc/release/upcoming_changes/15840.deprecation.rst +++ /dev/null @@ -1,6 +0,0 @@ -Deprecation of `round` for ``np.complexfloating`` scalars ------------------------------------------------------------ - -Output of the ``__round__`` dunder method and consequently the Python built-in -`round` has been deprecated on complex scalars. This does not affect -`np.round`. \ No newline at end of file diff --git a/doc/release/upcoming_changes/15867.deprecation.rst b/doc/release/upcoming_changes/15867.deprecation.rst deleted file mode 100644 index 79e3a638fe0e..000000000000 --- a/doc/release/upcoming_changes/15867.deprecation.rst +++ /dev/null @@ -1,6 +0,0 @@ -``numpy.ndarray.tostring()`` is deprecated in favor of ``tobytes()`` --------------------------------------------------------------------- -`~numpy.ndarray.tobytes` has existed since the 1.9 release, but until this -release `~numpy.ndarray.tostring` emitted no warning. The change to emit a -warning brings NumPy in line with the builtin `array.array` methods of the -same name. diff --git a/doc/release/upcoming_changes/15870.new_feature.rst b/doc/release/upcoming_changes/15870.new_feature.rst deleted file mode 100644 index 9da3f1535093..000000000000 --- a/doc/release/upcoming_changes/15870.new_feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -``keepdims`` parameter for `numpy.count_nonzero` ------------------------------------------------- -The parameter ``keepdims`` was added to `numpy.count_nonzero`. The -parameter has the same meaning as it does in reduction functions such -as `numpy.sum` or `numpy.mean`. diff --git a/doc/release/upcoming_changes/15872.change.rst b/doc/release/upcoming_changes/15872.change.rst deleted file mode 100644 index 7c48dee3c0d3..000000000000 --- a/doc/release/upcoming_changes/15872.change.rst +++ /dev/null @@ -1,6 +0,0 @@ -`Fixed `eigh` and `cholesky` methods in `numpy.random.multivariate_normal`` ---------------------------------------------------------------------------- - -Previously, when passing `method='eigh'` or `method='cholesky'`, -`numpy.random.multivariate_normal` produced samples from the wrong -distribution. This is now fixed. diff --git a/doc/release/upcoming_changes/15882.compatibility.rst b/doc/release/upcoming_changes/15882.compatibility.rst deleted file mode 100644 index 5790c081ceea..000000000000 --- a/doc/release/upcoming_changes/15882.compatibility.rst +++ /dev/null @@ -1,5 +0,0 @@ -The ``numpy.ndarray`` constructor no longer interprets ``strides=()`` as ``strides=None`` ------------------------------------------------------------------------------------------ -The former has changed to have the expected meaning of setting -`numpy.ndarray.strides` to ``()``, while the latter continues to result in -strides being chosen automatically. diff --git a/doc/release/upcoming_changes/16068.compatibility.rst b/doc/release/upcoming_changes/16068.compatibility.rst deleted file mode 100644 index 6fe693386d6f..000000000000 --- a/doc/release/upcoming_changes/16068.compatibility.rst +++ /dev/null @@ -1,11 +0,0 @@ -C-Level string to datetime casts changed ----------------------------------------- -The C-level casts from strings were simplified. This changed -also fixes string to datetime and timedelta casts to behave -correctly (i.e. like Python casts using ``string_arr.astype("M8")`` -while previously the cast would behave like -``string_arr.astype(np.int_).astype("M8")``. -This only affects code using low-level C-API to do manual casts -(not full array casts) of single scalar values or using e.g. -``PyArray_GetCastFunc``, and should thus not affect the vast majority -of users. diff --git a/doc/release/upcoming_changes/16080.improvement.rst b/doc/release/upcoming_changes/16080.improvement.rst deleted file mode 100644 index 6b1bd69e1fed..000000000000 --- a/doc/release/upcoming_changes/16080.improvement.rst +++ /dev/null @@ -1,4 +0,0 @@ -`numpy.einsum` accepts NumPy ``int64`` type in subscript list -------------------------------------------------------------- -There is no longer a type error thrown when `numpy.einsum` is passed -a NumPy ``int64`` array as its subscript list. \ No newline at end of file diff --git a/doc/release/upcoming_changes/16102.improvement.rst b/doc/release/upcoming_changes/16102.improvement.rst deleted file mode 100644 index cf81397f0c56..000000000000 --- a/doc/release/upcoming_changes/16102.improvement.rst +++ /dev/null @@ -1,4 +0,0 @@ -``np.logaddexp2.identity`` changed to ``-inf`` ----------------------------------------------- -The ufunc `~numpy.logaddexp2` now has an identity of ``-inf``, allowing it to -be called on empty sequences. This matches the identity of `~numpy.logaddexp`. diff --git a/doc/release/upcoming_changes/16128.new_feature.rst b/doc/release/upcoming_changes/16128.new_feature.rst deleted file mode 100644 index 03b061c07f01..000000000000 --- a/doc/release/upcoming_changes/16128.new_feature.rst +++ /dev/null @@ -1,6 +0,0 @@ -``equal_nan`` parameter for `numpy.array_equal` ------------------------------------------------- -The keyword argument ``equal_nan`` was added to `numpy.array_equal`. -``equal_nan`` is a boolean value that toggles whether or not ``nan`` values -are considered equal in comparison (default is ``False``). This matches API -used in related functions such as `numpy.isclose` and `numpy.allclose`. diff --git a/doc/release/upcoming_changes/16153.change.rst b/doc/release/upcoming_changes/16153.change.rst deleted file mode 100644 index 6612ff75d3d7..000000000000 --- a/doc/release/upcoming_changes/16153.change.rst +++ /dev/null @@ -1,10 +0,0 @@ -Fixed the jumping implementation in ``MT19937.jumped`` ------------------------------------------------------- - -This fix changes the stream produced from jumped MT19937 generators. It does -not affect the stream produced using ``RandomState`` or ``MT19937`` that -are directly seeded. - -The translation of the jumping code for the MT19937 contained a reversed loop -ordering. ``MT19937.jumped`` matches the Makoto Matsumoto's original -implementation of the Horner and Sliding Window jump methods. diff --git a/doc/release/upcoming_changes/8255.new_feature.rst b/doc/release/upcoming_changes/8255.new_feature.rst deleted file mode 100644 index c0bc21b3ef1e..000000000000 --- a/doc/release/upcoming_changes/8255.new_feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -`numpy.frompyfunc` now accepts an identity argument ---------------------------------------------------- -This allows the :attr:`numpy.ufunc.identity` attribute to be set on the -resulting ufunc, meaning it can be used for empty and multi-dimensional -calls to :meth:`numpy.ufunc.reduce`. diff --git a/doc/source/release.rst b/doc/source/release.rst index 1e56978286e1..5a890178c25e 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -5,6 +5,7 @@ Release Notes .. toctree:: :maxdepth: 3 + 1.20.0 1.19.0 1.18.4 1.18.3 diff --git a/doc/source/release/1.19.0-notes.rst b/doc/source/release/1.19.0-notes.rst index 6e7fd69d4167..ea0ac6193213 100644 --- a/doc/source/release/1.19.0-notes.rst +++ b/doc/source/release/1.19.0-notes.rst @@ -1,3 +1,472 @@ +========================== +NumPy 1.19.0 Release Notes +========================== + + +Highlights +========== + +* Code compatibility with Python versions < 3.5 (including Python 2) was + dropped from both the python and C code. The shims in numpy.compat will + remain to support third-party packages, but they may be deprecated in a + future release. + + (`gh-15233 `__) + + +Deprecations +============ + + +Deprecate automatic ``dtype=object`` for ragged input +----------------------------------------------------- +Calling ``np.array([[1, [1, 2, 3]])`` will issue a ``DeprecationWarning`` as +per `NEP 34`_. Users should explicitly use ``dtype=object`` to avoid the +warning. + +.. _`NEP 34`: https://numpy.org/neps/nep-0034.html + +(`gh-15119 `__) + +Passing ``shape=0`` to factory functions in ``numpy.rec`` is deprecated +----------------------------------------------------------------------- + +``0`` is treated as a special case and is aliased to ``None`` in the functions: + +* `numpy.core.records.fromarrays` +* `numpy.core.records.fromrecords` +* `numpy.core.records.fromstring` +* `numpy.core.records.fromfile` + +In future, ``0`` will not be special cased, and will be treated as an array +length like any other integer. + +(`gh-15217 `__) + +Deprecation of probably unused C-API functions +---------------------------------------------- +The following C-API functions are probably unused and have been +deprecated: + +* ``PyArray_GetArrayParamsFromObject`` +* ``PyUFunc_GenericFunction`` +* ``PyUFunc_SetUsesArraysAsData`` + +In most cases ``PyArray_GetArrayParamsFromObject`` should be replaced +by converting to an array, while ``PyUFunc_GenericFunction`` can be +replaced with ``PyObject_Call`` (see documentation for details). + +(`gh-15427 `__) + +Converting certain types to dtypes is Deprecated +------------------------------------------------ +The super classes of scalar types, such as ``np.integer``, ``np.generic``, +or ``np.inexact`` will now give a deprecation warning when converted +to a dtype (or used in a dtype keyword argument). +The reason for this is that `np.integer` is converted to ``np.int_``, +while it would be expected to represent *any* integer (e.g. also +``int8``, ``int16``, etc. +For example, ``dtype=np.floating`` is currently identical to +``dtype=np.float64``, even though also ``np.float32`` is a subclass of +``np.floating``. + +(`gh-15534 `__) + +Deprecation of `round` for ``np.complexfloating`` scalars +----------------------------------------------------------- + +Output of the ``__round__`` dunder method and consequently the Python built-in +`round` has been deprecated on complex scalars. This does not affect +`np.round`. + +(`gh-15840 `__) + +``numpy.ndarray.tostring()`` is deprecated in favor of ``tobytes()`` +-------------------------------------------------------------------- +`~numpy.ndarray.tobytes` has existed since the 1.9 release, but until this +release `~numpy.ndarray.tostring` emitted no warning. The change to emit a +warning brings NumPy in line with the builtin `array.array` methods of the +same name. + +(`gh-15867 `__) + + +Expired deprecations +==================== + +`numpy.insert` and `numpy.delete` can no longer be passed an axis on 0d arrays +------------------------------------------------------------------------------ +This concludes a deprecation from 1.9, where when an ``axis`` argument was +passed to a call to `~numpy.insert` and `~numpy.delete` on a 0d array, the +``axis`` and ``obj`` argument and indices would be completely ignored. +In these cases, ``insert(arr, "nonsense", 42, axis=0)`` would actually overwrite the +entire array, while ``delete(arr, "nonsense", axis=0)`` would be ``arr.copy()`` + +Now passing ``axis`` on a 0d array raises `~numpy.AxisError`. + +(`gh-15802 `__) + +`numpy.delete` no longer ignores out-of-bounds indices +------------------------------------------------------ +This concludes deprecations from 1.8 and 1.9, where ``np.delete`` would ignore +both negative and out-of-bounds items in a sequence of indices. This was at +odds with its behavior when passed a single index. + +Now out-of-bounds items throw ``IndexError``, and negative items index from the +end. + +(`gh-15804 `__) + +`numpy.insert` and `numpy.delete` no longer accept non-integral indices +----------------------------------------------------------------------- +This concludes a deprecation from 1.9, where sequences of non-integers indices +were allowed and cast to integers. Now passing sequences of non-integral +indices raises ``IndexError``, just like it does when passing a single +non-integral scalar. + +(`gh-15805 `__) + +`numpy.delete` no longer casts boolean indices to integers +---------------------------------------------------------- +This concludes a deprecation from 1.8, where ``np.delete`` would cast boolean +arrays and scalars passed as an index argument into integer indices. The +behavior now is to treat boolean arrays as a mask, and to raise an error +on boolean scalars. + +(`gh-15815 `__) + + +Compatibility notes +=================== + +Changed random variate stream from `numpy.random.Generator.dirichlet` +--------------------------------------------------------------------- +A bug in the generation of random variates for the Dirichlet distribution +with small `alpha` values was fixed by using a different algorithm when +``max(alpha) < 0.1``. Because of the change, the stream of variates +generated by `dirichlet` in this case will be different from previous +releases. + +(`gh-14924 `__) + +Scalar promotion in ``PyArray_ConvertToCommonType`` +--------------------------------------------------- + +The promotion of mixed scalars and arrays in ``PyArray_ConvertToCommonType`` +has been changed to adhere to those used by ``np.result_type``. +This means that input such as ``(1000, np.array([1], dtype=np.uint8)))`` +will now return ``uint16`` dtypes. In most cases the behaviour is unchanged. +Note that the use of this C-API function is generally discouraged. +This also fixes ``np.choose`` to behave the same way as the rest of NumPy +in this respect. + +(`gh-14933 `__) + +Fasttake and fastputmask slots are deprecated and NULL'ed +--------------------------------------------------------- +The fasttake and fastputmask slots are now never used and +must always be set to NULL. This will result in no change in behaviour. +However, if a user dtype should set one of these a DeprecationWarning +will be given. + +(`gh-14942 `__) + +`np.ediff1d` casting behaviour with ``to_end`` and ``to_begin`` +--------------------------------------------------------------- + +`np.ediff1d` now uses the ``"same_kind"`` casting rule for +its additional ``to_end`` and ``to_begin`` arguments. This +ensures type safety except when the input array has a smaller +integer type than ``to_begin`` or ``to_end``. +In rare cases, the behaviour will be more strict than it was +previously in 1.16 and 1.17. This is necessary to solve issues +with floating point NaN. + +(`gh-14981 `__) + +Converting of empty array-like objects to NumPy arrays +------------------------------------------------------ +Objects with ``len(obj) == 0`` which implement an "array-like" interface, +meaning an object implementing ``obj.__array__()``, +``obj.__array_interface__``, ``obj.__array_struct__``, or the python +buffer interface and which are also sequences (i.e. Pandas objects) +will now always retain there shape correctly when converted to an array. +If such an object has a shape of ``(0, 1)`` previously, it could +be converted into an array of shape ``(0,)`` (losing all dimensions +after the first 0). + +(`gh-14995 `__) + +Removed ``multiarray.int_asbuffer`` +----------------------------------- + +As part of the continued removal of Python 2 compatibility, +``multiarray.int_asbuffer`` was removed. On Python 3, it threw a +``NotImplementedError`` and was unused internally. It is expected that there +are no downstream use cases for this method with Python 3. + +(`gh-15229 `__) + +``numpy.distutils.compat`` has been removed +------------------------------------------- +This module contained only the function ``get_exception()``, which was used as:: + + try: + ... + except Exception: + e = get_exception() + +Its purpose was to handle the change in syntax introduced in Python 2.6, from +``except Exception, e:`` to ``except Exception as e:``, meaning it was only +necessary for codebases supporting Python 2.5 and older. + +(`gh-15255 `__) + +``issubdtype`` no longer interprets ``float`` as ``np.floating`` +---------------------------------------------------------------- + +`numpy.issubdtype` had a FutureWarning since NumPy 1.14 which +has expired now. This means that certain input where the second +argument was neither a datatype nor a NumPy scalar type +(such as a string or a python type like ``int`` or ``float``) +will now be consistent with passing in ``np.dtype(arg2).type``. +This makes the result consistent with expectations and leads to +a false result in some cases which previously returned true. + +(`gh-15773 `__) + +Change output of ``round`` on scalars to be consistent with Python +------------------------------------------------------------------ + +Output of the ``__round__`` dunder method and consequently the Python +built-in `round` has been changed to be a Python `int` to be consistent +with calling it on Python ``float`` objects when called with no arguments. +Previously, it would return a scalar of the `np.dtype` that was passed in. + +(`gh-15840 `__) + +The ``numpy.ndarray`` constructor no longer interprets ``strides=()`` as ``strides=None`` +----------------------------------------------------------------------------------------- +The former has changed to have the expected meaning of setting +`numpy.ndarray.strides` to ``()``, while the latter continues to result in +strides being chosen automatically. + +(`gh-15882 `__) + +C-Level string to datetime casts changed +---------------------------------------- +The C-level casts from strings were simplified. This changed +also fixes string to datetime and timedelta casts to behave +correctly (i.e. like Python casts using ``string_arr.astype("M8")`` +while previously the cast would behave like +``string_arr.astype(np.int_).astype("M8")``. +This only affects code using low-level C-API to do manual casts +(not full array casts) of single scalar values or using e.g. +``PyArray_GetCastFunc``, and should thus not affect the vast majority +of users. + +(`gh-16068 `__) + + +C API changes +============= + +Better support for ``const`` dimensions in API functions +-------------------------------------------------------- +The following functions now accept a constant array of ``npy_intp``: + +* `PyArray_BroadcastToShape` +* `PyArray_IntTupleFromIntp` +* `PyArray_OverflowMultiplyList` + +Previously the caller would have to cast away the const-ness to call these +functions. + +(`gh-15251 `__) + +Const qualify UFunc inner loops +------------------------------- +``UFuncGenericFunction`` now expects pointers to const ``dimension`` and +``strides`` as arguments. This means inner loops may no longer modify +either ``dimension`` or ``strides``. This change leads to an +``incompatible-pointer-types`` warning forcing users to either ignore +the compiler warnings or to const qualify their own loop signatures. + +(`gh-15355 `__) + + +New Features +============ + +`numpy.frompyfunc` now accepts an identity argument +--------------------------------------------------- +This allows the :attr:`numpy.ufunc.identity` attribute to be set on the +resulting ufunc, meaning it can be used for empty and multi-dimensional +calls to :meth:`numpy.ufunc.reduce`. + +(`gh-8255 `__) + +``np.str_`` scalars now support the buffer protocol +--------------------------------------------------- +``np.str_`` arrays are always stored as UCS4, so the corresponding scalars +now expose this through the buffer interface, meaning +``memoryview(np.str_('test'))`` now works. + +(`gh-15385 `__) + +``subok`` option for `numpy.copy` +--------------------------------- +A new kwarg, ``subok``, was added to `numpy.copy` to allow users to toggle the +behavior of `numpy.copy` with respect to array subclasses. The default value +is ``False`` which is consistent with the behavior of `numpy.copy` for +previous numpy versions. To create a copy that preserves an array subclass with +`numpy.copy`, call ``np.copy(arr, subok=True)``. This addition better documents +that the default behavior of `numpy.copy` differs from the +`numpy.ndarray.copy` method which respects array subclasses by default. + +(`gh-15685 `__) + +`numpy.linalg.multi_dot` now accepts an ``out`` argument +-------------------------------------------------------- + +``out`` can be used to avoid creating unnecessary copies of the final product +computed by `numpy.linalg.multidot`. + +(`gh-15715 `__) + +``keepdims`` parameter for `numpy.count_nonzero` +------------------------------------------------ +The parameter ``keepdims`` was added to `numpy.count_nonzero`. The +parameter has the same meaning as it does in reduction functions such +as `numpy.sum` or `numpy.mean`. + +(`gh-15870 `__) + +``equal_nan`` parameter for `numpy.array_equal` +------------------------------------------------ +The keyword argument ``equal_nan`` was added to `numpy.array_equal`. +``equal_nan`` is a boolean value that toggles whether or not ``nan`` values +are considered equal in comparison (default is ``False``). This matches API +used in related functions such as `numpy.isclose` and `numpy.allclose`. + +(`gh-16128 `__) + + +Improvements +============ + +Improve detection of CPU features +================================= + +Replace ``npy_cpu_supports`` which was a gcc-specific mechanism to test support +of avx with more general functions ``npy_cpu_init`` and ``npy_cpu_have``, and +expose the results via a ``NPY_CPU_HAVE`` c-macro as well as a python-level +``__cpu_features__`` dictionary. + +(`gh-13421 `__) + +Use 64-bit integer size on 64-bit platforms in fallback lapack_lite +------------------------------------------------------------------- + +Use 64-bit integer size on 64-bit platforms in the fallback LAPACK library, +which is used when the system has no LAPACK installed, allowing it to deal with +linear algebra for large arrays. + +(`gh-15218 `__) + +Use AVX512 intrinsic to implement ``np.exp`` when input is ``np.float64`` +-------------------------------------------------------------------------- +Use AVX512 intrinsic to implement ``np.exp`` when input is ``np.float64``, +which can improve the performance of ``np.exp`` with ``np.float64`` input 5-7x +faster than before. The _multiarray_umath.so module has grown about 63 KB on +linux64. + +(`gh-15648 `__) + +Ability to disable madvise hugepages +------------------------------------ + +On Linux NumPy has previously added support for madavise +hugepages which can improve performance for very large arrays. +Unfortunately, on older Kernel versions this led to peformance +regressions, thus by default the support has been disabled on +kernels before version 4.6. To override the default, you can +use the environment variable:: + + NUMPY_MADVISE_HUGEPAGE=0 + +or set it to 1 to force enabling support. Note that this only makes +a difference if the operating system is set up to use madvise +transparent hugepage. + +(`gh-15769 `__) + +`numpy.einsum` accepts NumPy ``int64`` type in subscript list +------------------------------------------------------------- +There is no longer a type error thrown when `numpy.einsum` is passed +a NumPy ``int64`` array as its subscript list. + +(`gh-16080 `__) + +``np.logaddexp2.identity`` changed to ``-inf`` +---------------------------------------------- +The ufunc `~numpy.logaddexp2` now has an identity of ``-inf``, allowing it to +be called on empty sequences. This matches the identity of `~numpy.logaddexp`. + +(`gh-16102 `__) + + +Changes +======= + +Remove handling of extra argument to ``__array__`` +-------------------------------------------------- +A code path and test have been in the code since NumPy 0.4 for a two-argument +variant of ``__array__(dtype=None, context=None)``. It was activated when +calling ``ufunc(op)`` or ``ufunc.reduce(op)`` if ``op.__array__`` existed. +However that variant is not documented, and it is not clear what the intention +was for its use. It has been removed. + +(`gh-15118 `__) + +``numpy.random._bit_generator`` moved to ``numpy.random.bit_generator`` +----------------------------------------------------------------------- + +In order to expose `numpy.random.BitGenerator` and `numpy.random.SeedSequence` +to cython, the ``_bitgenerator`` module is now public as +`numpy.random.bit_generator` + +Cython access to the random distributions is provided via a `pxd` file +---------------------------------------------------------------------- + +``c_distributions.pxd`` provides access to the c functions behind many of the +random distributions from Cython, making it convenient to use and extend them. + +(`gh-15463 `__) + +`Fixed `eigh` and `cholesky` methods in `numpy.random.multivariate_normal`` +--------------------------------------------------------------------------- + +Previously, when passing `method='eigh'` or `method='cholesky'`, +`numpy.random.multivariate_normal` produced samples from the wrong +distribution. This is now fixed. + +(`gh-15872 `__) + +Fixed the jumping implementation in ``MT19937.jumped`` +------------------------------------------------------ + +This fix changes the stream produced from jumped MT19937 generators. It does +not affect the stream produced using ``RandomState`` or ``MT19937`` that +are directly seeded. + +The translation of the jumping code for the MT19937 contained a reversed loop +ordering. ``MT19937.jumped`` matches the Makoto Matsumoto's original +implementation of the Horner and Sliding Window jump methods. + +(`gh-16153 `__) + + .. currentmodule:: numpy ========================== diff --git a/doc/source/release/1.20.0-notes.rst b/doc/source/release/1.20.0-notes.rst new file mode 100644 index 000000000000..d91bea7621a8 --- /dev/null +++ b/doc/source/release/1.20.0-notes.rst @@ -0,0 +1,6 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.20.0 Release Notes +========================== + diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 5daa52d79019..528113a9ed93 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -50,4 +50,5 @@ # Version 13 (NumPy 1.17) No change. # Version 13 (NumPy 1.18) No change. # Version 13 (NumPy 1.19) No change. +# Version 13 (NumPy 1.20) No change. 0x0000000d = 5b0e8bbded00b166125974fc71e80a33 diff --git a/numpy/core/include/numpy/numpyconfig.h b/numpy/core/include/numpy/numpyconfig.h index 4df4ea43820c..8eaf446b71d8 100644 --- a/numpy/core/include/numpy/numpyconfig.h +++ b/numpy/core/include/numpy/numpyconfig.h @@ -41,5 +41,6 @@ #define NPY_1_17_API_VERSION 0x00000008 #define NPY_1_18_API_VERSION 0x00000008 #define NPY_1_19_API_VERSION 0x00000008 +#define NPY_1_20_API_VERSION 0x00000008 #endif diff --git a/setup.py b/setup.py index 594c3ed91a8c..cb4f7801c195 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ """ MAJOR = 1 -MINOR = 19 +MINOR = 20 MICRO = 0 ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 4f88a0267a3a4992aacde45df7719fdef713bedc Mon Sep 17 00:00:00 2001 From: Dmitry Kutlenkov Date: Sun, 17 May 2020 20:08:37 +0300 Subject: [PATCH 0046/4003] DOC: Clarifications for np.std (#16267) * DOC: Clarifications for np.std * Line lengths fixes in np.std * Swapped variable names in np.std --- numpy/core/fromnumeric.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 285b753141ca..fa89418287ab 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -3436,17 +3436,18 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Notes ----- The standard deviation is the square root of the average of the squared - deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``. - - The average squared deviation is normally calculated as - ``x.sum() / N``, where ``N = len(x)``. If, however, `ddof` is specified, - the divisor ``N - ddof`` is used instead. In standard statistical - practice, ``ddof=1`` provides an unbiased estimator of the variance - of the infinite population. ``ddof=0`` provides a maximum likelihood - estimate of the variance for normally distributed variables. The - standard deviation computed in this function is the square root of - the estimated variance, so even with ``ddof=1``, it will not be an - unbiased estimate of the standard deviation per se. + deviations from the mean, i.e., ``std = sqrt(mean(x))``, where + ``x = abs(a - a.mean())**2``. + + The average squared deviation is typically calculated as ``x.sum() / N``, + where ``N = len(x)``. If, however, `ddof` is specified, the divisor + ``N - ddof`` is used instead. In standard statistical practice, ``ddof=1`` + provides an unbiased estimator of the variance of the infinite population. + ``ddof=0`` provides a maximum likelihood estimate of the variance for + normally distributed variables. The standard deviation computed in this + function is the square root of the estimated variance, so even with + ``ddof=1``, it will not be an unbiased estimate of the standard deviation + per se. Note that, for complex numbers, `std` takes the absolute value before squaring, so that the result is always real and nonnegative. From ec95e33b98e63b2b5e7bac31400c2c30998e052f Mon Sep 17 00:00:00 2001 From: Dmitry Kutlenkov Date: Sun, 17 May 2020 21:14:11 +0300 Subject: [PATCH 0047/4003] DOC: Clarifications for np.var --- numpy/core/fromnumeric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index fa89418287ab..7193af839c4f 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -3562,9 +3562,9 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Notes ----- The variance is the average of the squared deviations from the mean, - i.e., ``var = mean(abs(x - x.mean())**2)``. + i.e., ``var = mean(x)``, where ``x = abs(a - a.mean())**2``. - The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. + The mean is typically calculated as ``x.sum() / N``, where ``N = len(x)``. If, however, `ddof` is specified, the divisor ``N - ddof`` is used instead. In standard statistical practice, ``ddof=1`` provides an unbiased estimator of the variance of a hypothetical infinite population. From 38e3ca1382e10c272b2433be9935c24c263aca0b Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Mon, 18 May 2020 01:16:18 -0400 Subject: [PATCH 0048/4003] DOC: A few small changes in the "Using Python as Glue" page. * Fix whitespace in the inputs to Python in the f2py examples. * Capitalize Python and Fortran consistently. --- doc/source/user/c-info.python-as-glue.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/source/user/c-info.python-as-glue.rst b/doc/source/user/c-info.python-as-glue.rst index 6904a2fec89c..1a52b0385dbe 100644 --- a/doc/source/user/c-info.python-as-glue.rst +++ b/doc/source/user/c-info.python-as-glue.rst @@ -163,7 +163,7 @@ be imported from Python:: f2py -c -m add add.f This command leaves a file named add.{ext} in the current directory -(where {ext} is the appropriate extension for a python extension +(where {ext} is the appropriate extension for a Python extension module on your platform --- so, pyd, *etc.* ). This module may then be imported from Python. It will contain a method for each subroutine in add (zadd, cadd, dadd, sadd). The docstring of each method contains @@ -185,7 +185,7 @@ information about how the module method may be called:: Improving the basic interface ----------------------------- -The default interface is a very literal translation of the fortran +The default interface is a very literal translation of the Fortran code into Python. The Fortran array arguments must now be NumPy arrays and the integer argument should be an integer. The interface will attempt to convert all arguments to their required types (and shapes) @@ -194,7 +194,7 @@ about the semantics of the arguments (such that C is an output and n should really match the array sizes), it is possible to abuse this function in ways that can cause Python to crash. For example:: - >>> add.zadd([1,2,3], [1,2], [3,4], 1000) + >>> add.zadd([1, 2, 3], [1, 2], [3, 4], 1000) will cause a program crash on most systems. Under the covers, the lists are being converted to proper arrays but then the underlying add @@ -242,7 +242,7 @@ necessary to tell f2py that the value of n depends on the input a (so that it won't try to create the variable n until the variable a is created). -After modifying ``add.pyf``, the new python module file can be generated +After modifying ``add.pyf``, the new Python module file can be generated by compiling both ``add.f`` and ``add.pyf``:: f2py -c add.pyf add.f @@ -266,7 +266,7 @@ The new interface has docstring:: Now, the function can be called in a much more robust way:: - >>> add.zadd([1,2,3],[4,5,6]) + >>> add.zadd([1, 2, 3], [4, 5, 6]) array([5.+0.j, 7.+0.j, 9.+0.j]) Notice the automatic conversion to the correct format that occurred. @@ -276,7 +276,7 @@ Inserting directives in Fortran source -------------------------------------- The nice interface can also be generated automatically by placing the -variable directives as special comments in the original fortran code. +variable directives as special comments in the original Fortran code. Thus, if I modify the source code to contain: .. code-block:: none @@ -662,7 +662,7 @@ To use ctypes you must 2. Load the shared library. -3. Convert the python objects to ctypes-understood arguments. +3. Convert the Python objects to ctypes-understood arguments. 4. Call the function from the library with the ctypes arguments. From 6ce44a4068d0971092815049d7ab030c65945af5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 09:03:52 +0000 Subject: [PATCH 0049/4003] MAINT: Bump hypothesis from 5.12.0 to 5.14.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.12.0 to 5.14.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.12.0...hypothesis-python-5.14.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 5db322b9bed1..607fabe1e2aa 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ cython==0.29.17 -hypothesis==5.12.0 +hypothesis==5.14.0 pytest==5.4.2 pytz==2020.1 pytest-cov==2.8.1 From 0eb532de60c9caaaa568ea294e7e1427265b6552 Mon Sep 17 00:00:00 2001 From: Raphael Kruse <65403623+rk-mlu@users.noreply.github.com> Date: Mon, 18 May 2020 18:22:29 +0200 Subject: [PATCH 0050/4003] DOC: Calrify tiny/xmin in finfo and machar (gh-16253) This commit adds a note to the documentation of finfo clarifying that the attribute `tiny` does not represent the smallest positive usable number. Instead it refers to the smallest positive number with there being no leading 0's in the mantisse (normalized floating point number). Alternatively, `tiny` represents the smallest positive floating point number with full precision. The smallest positive subnormal floating point number is typically orders of magnitudes smaller, but has reduced precision since leading 0's in the mantisse are used to allow smaller exponents than indicated by `minexp`. The commit also adds a brief clarification to the docstring of machar. Closes #16252 Co-authored-by: Anirudh Subramanian Co-authored-by: Ross Barnowski --- numpy/core/getlimits.py | 17 ++++++++++++++--- numpy/core/machar.py | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/numpy/core/getlimits.py b/numpy/core/getlimits.py index e2ff493932d7..f73c21f67a9f 100644 --- a/numpy/core/getlimits.py +++ b/numpy/core/getlimits.py @@ -337,8 +337,8 @@ class finfo: The approximate decimal resolution of this type, i.e., ``10**-precision``. tiny : float - The smallest positive usable number. Type of `tiny` is an - appropriate floating point type. + The smallest positive floating point number with full precision + (see Notes). Parameters ---------- @@ -359,6 +359,18 @@ class finfo: impacts import times. These objects are cached, so calling ``finfo()`` repeatedly inside your functions is not a problem. + Note that ``tiny`` is not actually the smallest positive representable + value in a NumPy floating point type. As in the IEEE-754 standard [1]_, + NumPy floating point types make use of subnormal numbers to fill the + gap between 0 and ``tiny``. However, subnormal numbers may have + significantly reduced precision [2]_. + + References + ---------- + .. [1] IEEE Standard for Floating-Point Arithmetic, IEEE Std 754-2008, + pp.1-70, 2008, http://www.doi.org/10.1109/IEEESTD.2008.4610935 + .. [2] Wikipedia, "Denormal Numbers", + https://en.wikipedia.org/wiki/Denormal_number """ _finfo_cache = {} @@ -546,4 +558,3 @@ def __str__(self): def __repr__(self): return "%s(min=%s, max=%s, dtype=%s)" % (self.__class__.__name__, self.min, self.max, self.dtype) - diff --git a/numpy/core/machar.py b/numpy/core/machar.py index a48dc3d50df4..55285fe5928f 100644 --- a/numpy/core/machar.py +++ b/numpy/core/machar.py @@ -40,8 +40,8 @@ class MachAr: Smallest (most negative) power of `ibeta` consistent with there being no leading zeros in the mantissa. xmin : float - Floating point number ``beta**minexp`` (the smallest [in - magnitude] usable floating value). + Floating-point number ``beta**minexp`` (the smallest [in + magnitude] positive floating point number with full precision). maxexp : int Smallest (positive) power of `ibeta` that causes overflow. xmax : float From 19efe2199845d1ba5464795eed89a347c70d22d8 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Mon, 18 May 2020 13:31:56 -0300 Subject: [PATCH 0051/4003] DOC: Add PyArray_ContiguousFromObject C docs (gh-16207) Add documentation for `PyArray_ContiguousFromObject` similar to the other macros wrapping `PyArray_FromAny` Fixes #16196 --- doc/source/reference/c-api/array.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 56fc59da39d2..10c1704c26ac 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -251,7 +251,6 @@ From scratch .. versionadded:: 1.6 This function steals a reference to *descr* if it is not NULL. - This array creation routine allows for the convenient creation of a new array matching an existing array's shapes and memory layout, possibly changing the layout and/or data type. @@ -634,8 +633,17 @@ From other objects requirements set to :c:data:`NPY_ARRAY_DEFAULT` and the type_num member of the type argument set to *typenum*. -.. c:function:: PyObject *PyArray_FromObject( \ - PyObject *op, int typenum, int min_depth, int max_depth) +.. c:function:: PyObject* PyArray_ContiguousFromObject( \ + PyObject* op, int typenum, int min_depth, int max_depth) + + This function returns a well-behaved C-style contiguous array from any nested + sequence or array-interface exporting object. The minimum number of dimensions + the array can have is given by `min_depth` while the maximum is `max_depth`. + This is equivalent to call :c:func:`PyArray_FromAny` with requirements + :c:data:`NPY_ARRAY_DEFAULT` and :c:data:`NPY_ARRAY_ENSUREARRAY`. + +.. c:function:: PyObject* PyArray_FromObject( \ + PyObject* op, int typenum, int min_depth, int max_depth) Return an aligned and in native-byteorder array from any nested sequence or array-interface exporting object, op, of a type given by From 625f3345189ed4742ecde22d569398062d4743d2 Mon Sep 17 00:00:00 2001 From: abhilash42 Date: Mon, 18 May 2020 23:41:49 +0530 Subject: [PATCH 0052/4003] Update _add_newdocs.py --- numpy/core/_add_newdocs.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index f43b77c44c08..1343c6bc3fa3 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -1525,7 +1525,7 @@ Controls the memory layout of the output. 'C' means it should be C contiguous. 'F' means it should be Fortran contiguous, 'A' means it should be 'F' if the inputs are all 'F', 'C' otherwise. - 'K' means it should be as close to the layout as the inputs as + 'K' means it should be as close to the layout of the inputs as is possible, including arbitrarily permuted axes. Default is 'K'. casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional @@ -3939,9 +3939,10 @@ Constructs Python bytes showing a copy of the raw contents of data memory. The bytes object can be produced in either 'C' or 'Fortran', - or 'Any' order (the default is 'C'-order). 'Any' order means C-order + or 'K' or 'A' order (the default is 'C'-order). 'A' order means C-order unless the F_CONTIGUOUS flag in the array is set, in which case it - means 'Fortran' order. + means 'Fortran' order. 'K' order is as close to the order array elements appear + in memory. .. versionadded:: 1.9.0 @@ -3949,7 +3950,11 @@ ---------- order : {'C', 'F', None}, optional Order of the data for multidimensional arrays: - C, Fortran, or the same as for the original array. + 'C' means it should be C contiguous, + 'F' means it should be Fortran contiguous, + 'A' means it should be 'F' if the inputs are all 'F', 'C' otherwise. + 'K' means it should be as close to the layout of the inputs as + is possible, including arbitrarily permuted axes. Returns ------- @@ -3965,6 +3970,10 @@ True >>> x.tobytes('F') b'\\x00\\x00\\x02\\x00\\x01\\x00\\x03\\x00' + >>> x.tobytes('A') == x.tobytes() + True + >>> x.tobytes('K') == x.tobytes() + True """)) From 57a857d1793107eb96b559a9a9071fa6187c3679 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 18 May 2020 12:25:47 -0600 Subject: [PATCH 0053/4003] BUG: Fix tools/download-wheels.py. `tools/download-wheels` was downloading an html page rather than the binary wheel file. --- tools/download-wheels.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/download-wheels.py b/tools/download-wheels.py index 68437d129cce..276fcc6b268f 100644 --- a/tools/download-wheels.py +++ b/tools/download-wheels.py @@ -14,9 +14,9 @@ __version__ = '0.1' -ANACONDA_INDEX = 'https://anaconda.org/multibuild-wheels-staging/numpy/files' -ANACONDA_FILES = 'https://anaconda.org/multibuild-wheels-staging/numpy/simple' - +# Edit these for other projects. +STAGING_URL = 'https://anaconda.org/multibuild-wheels-staging/numpy' +PREFIX = '^.*numpy-' def get_wheel_names(version): """ Get wheel names from Anaconda HTML directory. @@ -30,10 +30,11 @@ def get_wheel_names(version): The release version. For instance, "1.18.3". """ - tmpl = re.compile('^.*numpy-' + version + '.*\.whl$') http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED') - indx = http.request('GET', ANACONDA_INDEX) - soup = BeautifulSoup(indx.data, 'html.parser') + tmpl = re.compile(PREFIX + version + '.*\.whl$') + index_url = f"{STAGING_URL}/files" + index_html = http.request('GET', index_url) + soup = BeautifulSoup(index_html.data, 'html.parser') return soup.findAll(text=tmpl) @@ -51,10 +52,10 @@ def download_wheels(version, wheelhouse): Directory in which to download the wheels. """ - wheel_names = get_wheel_names(version[0]) http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED') + wheel_names = get_wheel_names(version) for wheel_name in wheel_names: - wheel_url = os.path.join(ANACONDA_FILES, wheel_name) + wheel_url = f"{STAGING_URL}/{version}/download/{wheel_name}" wheel_path = os.path.join(wheelhouse, wheel_name) with open(wheel_path, 'wb') as f: with http.request('GET', wheel_url, preload_content=False,) as r: From 91f147091316595be3b875ca9de2361666318370 Mon Sep 17 00:00:00 2001 From: abhilash42 Date: Tue, 19 May 2020 00:56:30 +0530 Subject: [PATCH 0054/4003] Update _add_newdocs.py --- numpy/core/_add_newdocs.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 1343c6bc3fa3..85b19091df35 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3938,23 +3938,19 @@ Construct Python bytes containing the raw data bytes in the array. Constructs Python bytes showing a copy of the raw contents of - data memory. The bytes object can be produced in either 'C' or 'Fortran', - or 'K' or 'A' order (the default is 'C'-order). 'A' order means C-order - unless the F_CONTIGUOUS flag in the array is set, in which case it - means 'Fortran' order. 'K' order is as close to the order array elements appear - in memory. + data memory. The bytes object can be produced in C-order by default. + This behavior is controlled by the order parameter. .. versionadded:: 1.9.0 Parameters ---------- - order : {'C', 'F', None}, optional - Order of the data for multidimensional arrays: - 'C' means it should be C contiguous, - 'F' means it should be Fortran contiguous, - 'A' means it should be 'F' if the inputs are all 'F', 'C' otherwise. - 'K' means it should be as close to the layout of the inputs as - is possible, including arbitrarily permuted axes. + order : {'C', 'F', 'A', 'K'}, optional + Controls the memory layout of the copy. 'C' means C-order, + 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, + 'C' otherwise. 'K' means match the layout of `a` as closely + as possible. + Default is 'C'. Returns ------- @@ -3970,10 +3966,6 @@ True >>> x.tobytes('F') b'\\x00\\x00\\x02\\x00\\x01\\x00\\x03\\x00' - >>> x.tobytes('A') == x.tobytes() - True - >>> x.tobytes('K') == x.tobytes() - True """)) From eb30df43d7c6fc472c2b806df05078611fb6af18 Mon Sep 17 00:00:00 2001 From: abhilash42 <64172584+abhilash42@users.noreply.github.com> Date: Tue, 19 May 2020 01:42:12 +0530 Subject: [PATCH 0055/4003] Update numpy/core/_add_newdocs.py Co-authored-by: Ross Barnowski --- numpy/core/_add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 85b19091df35..470bc5febca2 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3938,7 +3938,7 @@ Construct Python bytes containing the raw data bytes in the array. Constructs Python bytes showing a copy of the raw contents of - data memory. The bytes object can be produced in C-order by default. + data memory. The bytes object is produced in C-order by default. This behavior is controlled by the order parameter. .. versionadded:: 1.9.0 From 44632dff703dfd1e9afabf6c407be7bc77296a76 Mon Sep 17 00:00:00 2001 From: abhilash42 <64172584+abhilash42@users.noreply.github.com> Date: Tue, 19 May 2020 01:42:26 +0530 Subject: [PATCH 0056/4003] Update numpy/core/_add_newdocs.py Co-authored-by: Ross Barnowski --- numpy/core/_add_newdocs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 470bc5febca2..0a818dfb80c8 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3949,8 +3949,7 @@ Controls the memory layout of the copy. 'C' means C-order, 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, 'C' otherwise. 'K' means match the layout of `a` as closely - as possible. - Default is 'C'. + as possible. Default is 'C'. Returns ------- From 637ecfeed67b084fcdbfabd938ae23dbe01276b3 Mon Sep 17 00:00:00 2001 From: abhilash42 <64172584+abhilash42@users.noreply.github.com> Date: Tue, 19 May 2020 01:42:52 +0530 Subject: [PATCH 0057/4003] Update numpy/core/_add_newdocs.py Co-authored-by: Ross Barnowski --- numpy/core/_add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 0a818dfb80c8..6562cb6dc751 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3939,7 +3939,7 @@ Constructs Python bytes showing a copy of the raw contents of data memory. The bytes object is produced in C-order by default. - This behavior is controlled by the order parameter. + This behavior is controlled by the ``order`` parameter. .. versionadded:: 1.9.0 From ccfe54db8b187a2683055fb98aa6a8119fc7f0b6 Mon Sep 17 00:00:00 2001 From: abhilash42 <64172584+abhilash42@users.noreply.github.com> Date: Tue, 19 May 2020 01:47:08 +0530 Subject: [PATCH 0058/4003] Update numpy/core/_add_newdocs.py Co-authored-by: Ross Barnowski --- numpy/core/_add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 6562cb6dc751..ae8bc965e388 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3946,7 +3946,7 @@ Parameters ---------- order : {'C', 'F', 'A', 'K'}, optional - Controls the memory layout of the copy. 'C' means C-order, + Controls the memory layout of the bytes object. 'C' means C-order, 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, 'C' otherwise. 'K' means match the layout of `a` as closely as possible. Default is 'C'. From 5e8ad11b75245caf2d158eacf369b011ef262795 Mon Sep 17 00:00:00 2001 From: lkdmttg7 Date: Tue, 19 May 2020 02:48:39 +0530 Subject: [PATCH 0059/4003] MAINT: Chain exceptions in generate_umath.py (gh-16254) Co-authored-by: Sebastian Berg Co-authored-by: Eric Wieser --- numpy/core/code_generators/generate_umath.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 202912c5f31c..f10ce9f0f62e 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -1028,8 +1028,11 @@ def make_arrays(funcdict): funclist.append('NULL') try: thedict = arity_lookup[uf.nin, uf.nout] - except KeyError: - raise ValueError("Could not handle {}[{}]".format(name, t.type)) + except KeyError as e: + raise ValueError( + f"Could not handle {name}[{t.type}] " + f"with nin={uf.nin}, nout={uf.nout}" + ) from None astype = '' if not t.astype is None: From f70c2c507f3664901e97754e98b8e30e8e39b85d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 17 May 2020 15:26:25 +0100 Subject: [PATCH 0060/4003] MAINT: Extract a lerp helper function to make the algorithm of quantile clearer This does not affect the behavior in any way --- numpy/lib/function_base.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 2d1adc362f61..9b547d82318d 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3870,6 +3870,11 @@ def _quantile_is_valid(q): return True +def _lerp(a, b, t, out=None): + """ Linearly interpolate from a to b by a factor of t """ + return add(a*(1 - t), b*t, out=out) + + def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, interpolation='linear', keepdims=False): a = asarray(a) @@ -3958,14 +3963,14 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, indices_above = indices_above[:-1] n = np.isnan(ap[-1:, ...]) - x1 = take(ap, indices_below, axis=axis) * (1 - weights_above) - x2 = take(ap, indices_above, axis=axis) * weights_above - + x1 = take(ap, indices_below, axis=axis) + x2 = take(ap, indices_above, axis=axis) if zerod: x1 = x1.squeeze(0) x2 = x2.squeeze(0) + weights_above = weights_above.squeeze(0) - r = add(x1, x2, out=out) + r = _lerp(x1, x2, weights_above, out=out) if np.any(n): if zerod: From 919a8258da892cb154c29b83521b913630820fda Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 17 May 2020 15:56:16 +0100 Subject: [PATCH 0061/4003] MAINT: Remove special cases for 0d arrays in quantile This also simplifies the axis handling logic, taking advantage of the fact we know `axis` is 0 for the rest of the function body --- numpy/lib/function_base.py | 117 +++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 62 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 9b547d82318d..00f72acba409 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3878,12 +3878,12 @@ def _lerp(a, b, t, out=None): def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, interpolation='linear', keepdims=False): a = asarray(a) - if q.ndim == 0: - # Do not allow 0-d arrays because following code fails for scalar - zerod = True - q = q[None] - else: - zerod = False + + # ufuncs cause 0d array results to decay to scalars (see gh-13105), which + # makes them problematic for __setitem__ and attribute access. As a + # workaround, we call this on the result of every ufunc on a possibly-0d + # array. + not_scalar = np.asanyarray # prepare a for partitioning if overwrite_input: @@ -3900,9 +3900,14 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, if axis is None: axis = 0 - Nx = ap.shape[axis] - indices = q * (Nx - 1) + if q.ndim > 2: + # The code below works fine for nd, but it might not have useful + # semantics. For now, keep the supported dimensions the same as it was + # before. + raise ValueError("q must be a scalar or 1d") + Nx = ap.shape[axis] + indices = not_scalar(q * (Nx - 1)) # round fractional indices according to interpolation method if interpolation == 'lower': indices = floor(indices).astype(intp) @@ -3919,74 +3924,62 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, "interpolation can only be 'linear', 'lower' 'higher', " "'midpoint', or 'nearest'") - n = np.array(False, dtype=bool) # check for nan's flag - if np.issubdtype(indices.dtype, np.integer): # take the points along axis - # Check if the array contains any nan's - if np.issubdtype(a.dtype, np.inexact): - indices = concatenate((indices, [-1])) + # The dimensions of `q` are prepended to the output shape, so we need the + # axis being sampled from `ap` to be first. + ap = np.moveaxis(ap, axis, 0) + del axis - ap.partition(indices, axis=axis) - # ensure axis with q-th is first - ap = np.moveaxis(ap, axis, 0) - axis = 0 + if np.issubdtype(indices.dtype, np.integer): + # take the points along axis - # Check if the array contains any nan's if np.issubdtype(a.dtype, np.inexact): - indices = indices[:-1] - n = np.isnan(ap[-1:, ...]) + # may contain nan, which would sort to the end + ap.partition(concatenate((indices.ravel(), [-1])), axis=0) + n = np.isnan(ap[-1]) + else: + # cannot contain nan + ap.partition(indices.ravel(), axis=0) + n = np.array(False, dtype=bool) - if zerod: - indices = indices[0] - r = take(ap, indices, axis=axis, out=out) + r = take(ap, indices, axis=0, out=out) + if out is not None: + r = out # workaround for gh-16276 + + else: + # weight the points above and below the indices - else: # weight the points above and below the indices - indices_below = floor(indices).astype(intp) - indices_above = indices_below + 1 + indices_below = not_scalar(floor(indices)).astype(intp) + indices_above = not_scalar(indices_below + 1) indices_above[indices_above > Nx - 1] = Nx - 1 - # Check if the array contains any nan's if np.issubdtype(a.dtype, np.inexact): - indices_above = concatenate((indices_above, [-1])) - - ap.partition(concatenate((indices_below, indices_above)), axis=axis) - - # ensure axis with q-th is first - ap = np.moveaxis(ap, axis, 0) - axis = 0 - - weights_shape = [1] * ap.ndim - weights_shape[axis] = len(indices) - weights_above = (indices - indices_below).reshape(weights_shape) + # may contain nan, which would sort to the end + ap.partition(concatenate(( + indices_below.ravel(), indices_above.ravel(), [-1] + )), axis=0) + n = np.isnan(ap[-1]) + else: + # cannot contain nan + ap.partition(concatenate(( + indices_below.ravel(), indices_above.ravel() + )), axis=0) + n = np.array(False, dtype=bool) - # Check if the array contains any nan's - if np.issubdtype(a.dtype, np.inexact): - indices_above = indices_above[:-1] - n = np.isnan(ap[-1:, ...]) + weights_shape = indices.shape + (1,) * (ap.ndim - 1) + weights_above = not_scalar(indices - indices_below).reshape(weights_shape) - x1 = take(ap, indices_below, axis=axis) - x2 = take(ap, indices_above, axis=axis) - if zerod: - x1 = x1.squeeze(0) - x2 = x2.squeeze(0) - weights_above = weights_above.squeeze(0) + x_below = take(ap, indices_below, axis=0) + x_above = take(ap, indices_above, axis=0) - r = _lerp(x1, x2, weights_above, out=out) + r = _lerp(x_below, x_above, weights_above, out=out) + # if any slice contained a nan, then all results on that slice are also nan if np.any(n): - if zerod: - if ap.ndim == 1: - if out is not None: - out[...] = a.dtype.type(np.nan) - r = out - else: - r = a.dtype.type(np.nan) - else: - r[..., n.squeeze(0)] = a.dtype.type(np.nan) + if r.ndim == 0 and out is None: + # can't write to a scalar + r = a.dtype.type(np.nan) else: - if r.ndim == 1: - r[:] = a.dtype.type(np.nan) - else: - r[..., n.repeat(q.size, 0)] = a.dtype.type(np.nan) + r[..., n] = a.dtype.type(np.nan) return r From e73fdb402c13da019eeb3de4d67002d81a335801 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 17 May 2020 16:07:31 +0100 Subject: [PATCH 0062/4003] BUG: Ensure out argument is returned by identity for 0d arrays This makes the following functions consistent with the behavior of ufuncs: * `np.argmax` * `np.argmin` * `np.choose` * `np.trace` * `np.round` * `np.take` Previously they would unpack these into scalars. --- numpy/core/src/multiarray/methods.c | 74 +++++++++++++++++++++++++---- numpy/core/tests/test_multiarray.py | 35 +++++++++++++- 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index e2026ec1c3ab..3679a34b86fb 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -146,8 +146,15 @@ array_take(PyArrayObject *self, PyObject *args, PyObject *kwds) PyArray_ClipmodeConverter, &mode)) return NULL; - return PyArray_Return((PyArrayObject *) - PyArray_TakeFrom(self, indices, dimension, out, mode)); + PyObject *ret = PyArray_TakeFrom(self, indices, dimension, out, mode); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } static PyObject * @@ -303,7 +310,15 @@ array_argmax(PyArrayObject *self, PyObject *args, PyObject *kwds) PyArray_OutputConverter, &out)) return NULL; - return PyArray_Return((PyArrayObject *)PyArray_ArgMax(self, axis, out)); + PyObject *ret = PyArray_ArgMax(self, axis, out); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } static PyObject * @@ -318,7 +333,15 @@ array_argmin(PyArrayObject *self, PyObject *args, PyObject *kwds) PyArray_OutputConverter, &out)) return NULL; - return PyArray_Return((PyArrayObject *)PyArray_ArgMin(self, axis, out)); + PyObject *ret = PyArray_ArgMin(self, axis, out); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } static PyObject * @@ -1218,7 +1241,15 @@ array_choose(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - return PyArray_Return((PyArrayObject *)PyArray_Choose(self, choices, out, clipmode)); + PyObject *ret = PyArray_Choose(self, choices, out, clipmode); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } static PyObject * @@ -2319,8 +2350,16 @@ array_compress(PyArrayObject *self, PyObject *args, PyObject *kwds) PyArray_OutputConverter, &out)) { return NULL; } - return PyArray_Return( - (PyArrayObject *)PyArray_Compress(self, condition, axis, out)); + + PyObject *ret = PyArray_Compress(self, condition, axis, out); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } @@ -2355,7 +2394,15 @@ array_trace(PyArrayObject *self, PyObject *args, PyObject *kwds) rtype = _CHKTYPENUM(dtype); Py_XDECREF(dtype); - return PyArray_Return((PyArrayObject *)PyArray_Trace(self, offset, axis1, axis2, rtype, out)); + PyObject *ret = PyArray_Trace(self, offset, axis1, axis2, rtype, out); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } #undef _CHKTYPENUM @@ -2440,7 +2487,16 @@ array_round(PyArrayObject *self, PyObject *args, PyObject *kwds) PyArray_OutputConverter, &out)) { return NULL; } - return PyArray_Return((PyArrayObject *)PyArray_Round(self, decimals, out)); + + PyObject *ret = PyArray_Round(self, decimals, out); + + /* this matches the unpacking behavior of ufuncs */ + if (out == NULL) { + return PyArray_Return((PyArrayObject *)ret); + } + else { + return ret; + } } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index f36c27c6cb19..a698370b6950 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1582,6 +1582,11 @@ def test_choose(self): # gh-12031, caused SEGFAULT assert_raises(TypeError, oned.choose,np.void(0), [oned]) + out = np.array(0) + ret = np.choose(np.array(1), [10, 20, 30], out=out) + assert out is ret + assert_equal(out[()], 20) + # gh-6272 check overlap on out x = np.arange(5) y = np.choose([0,0,0], [x[:3], x[:3], x[:3]], out=x[1:4], mode='wrap') @@ -1658,7 +1663,7 @@ def check_round(arr, expected, *round_args): out = np.zeros_like(arr) res = arr.round(*round_args, out=out) assert_equal(out, expected) - assert_equal(out, res) + assert out is res check_round(np.array([1.2, 1.5]), [1, 2]) check_round(np.array(1.5), 2) @@ -3023,6 +3028,10 @@ def test_trace(self): assert_equal(b.trace(0, 1, 2), [3, 11]) assert_equal(b.trace(offset=1, axis1=0, axis2=2), [1, 3]) + out = np.array(1) + ret = a.trace(out=out) + assert ret is out + def test_trace_subclass(self): # The class would need to overwrite trace to ensure single-element # output also has the right subclass. @@ -4126,6 +4135,13 @@ def test_output_shape(self): a.argmax(-1, out=out) assert_equal(out, a.argmax(-1)) + @pytest.mark.parametrize('ndim', [0, 1]) + def test_ret_is_out(self, ndim): + a = np.ones((4,) + (3,)*ndim) + out = np.empty((3,)*ndim, dtype=np.intp) + ret = a.argmax(axis=0, out=out) + assert ret is out + def test_argmax_unicode(self): d = np.zeros(6031, dtype=' Date: Thu, 14 May 2020 15:55:41 -0500 Subject: [PATCH 0063/4003] BUG: Fix default fallback in genfromtxt This affected (for example?) if the `dtype=object` was used without a converter, meaning that the default one is used. And this is currently the last one, which is `string_` (and thus bytes). Closes gh-16189 --- numpy/lib/_iotools.py | 8 +++++--- numpy/lib/tests/test__iotools.py | 4 ++-- numpy/lib/tests/test_io.py | 7 +++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 84aff5e5d75b..ac5f4f9b422a 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -506,13 +506,15 @@ class StringConverter: _mapper.extend([(nx.float64, float, nx.nan), (nx.complex128, complex, nx.nan + 0j), (nx.longdouble, nx.longdouble, nx.nan), - (nx.unicode_, asunicode, '???'), - (nx.string_, asbytes, '???'), # If a non-default dtype is passed, fall back to generic # ones (should only be used for the converter) (nx.integer, int, -1), (nx.floating, float, nx.nan), - (nx.complexfloating, complex, nx.nan + 0j),]) + (nx.complexfloating, complex, nx.nan + 0j), + # Last, try with the string types (must be last, because + # `_mapper[-1]` is used as default in some cases) + (nx.unicode_, asunicode, '???'), + (nx.string_, asbytes, '???'),]) @classmethod def _getdtype(cls, val): diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index 6964c11280e6..a5b78702525e 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -177,12 +177,12 @@ def test_upgrade(self): # test str # note that the longdouble type has been skipped, so the # _status increases by 2. Everything should succeed with - # unicode conversion (5). + # unicode conversion (8). for s in ['a', b'a']: res = converter.upgrade(s) assert_(type(res) is str) assert_equal(res, 'a') - assert_equal(converter._status, 5 + status_offset) + assert_equal(converter._status, 8 + status_offset) def test_missing(self): "Tests the use of missing values." diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 9abde3e11f6b..99d119362b8f 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -1567,6 +1567,13 @@ def test_dtype_with_object(self): test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype, converters=converters) + def test_dtype_with_object_no_converter(self): + # Object without a converter uses bytes: + parsed = np.genfromtxt(TextIO("1"), dtype=object) + assert parsed[()] == b"1" + parsed = np.genfromtxt(TextIO("string"), dtype=object) + assert parsed[()] == b"string" + def test_userconverters_with_explicit_dtype(self): # Test user_converters w/ explicit (standard) dtype data = TextIO('skip,skip,2001-01-01,1.0,skip') From 4b401344f6bc93a466faf4f88264608a06c43cb9 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 18 May 2020 16:50:58 -0500 Subject: [PATCH 0064/4003] Update numpy/lib/_iotools.py Co-authored-by: Eric Wieser --- numpy/lib/_iotools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index ac5f4f9b422a..7560bf4daab4 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -514,7 +514,8 @@ class StringConverter: # Last, try with the string types (must be last, because # `_mapper[-1]` is used as default in some cases) (nx.unicode_, asunicode, '???'), - (nx.string_, asbytes, '???'),]) + (nx.string_, asbytes, '???'), + ]) @classmethod def _getdtype(cls, val): From b3b6cc0347979a21b48ca22654b1caf9387045b0 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 18 May 2020 16:26:10 -0600 Subject: [PATCH 0065/4003] BUG: Require Python >= 3.6 in setup.py Closes #16294. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cb4f7801c195..a86e28341145 100755 --- a/setup.py +++ b/setup.py @@ -443,9 +443,9 @@ def setup_package(): license = 'BSD', classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], - test_suite='nose.collector', + test_suite='pytest', cmdclass=cmdclass, - python_requires='>=3.5', + python_requires='>=3.6', zip_safe=False, entry_points={ 'console_scripts': f2py_cmds From 74c111f0c40978ad064daabfe17658a308721a2d Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 19 May 2020 17:05:21 +1000 Subject: [PATCH 0066/4003] DOC: Add math.isclose to See Also (#16283) * DOC: Move math.isclose note to See Also. --- numpy/core/numeric.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 8bd4e241bceb..05f0b7820689 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -2230,6 +2230,7 @@ def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): See Also -------- allclose + math.isclose Notes ----- From c4a1aef1c9d961907e346cab809e39b97172cadc Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 19 May 2020 10:16:50 +0100 Subject: [PATCH 0067/4003] BUG: np.info does not show keyword-only arguments Using inspect.signature instead of `np.compat.getargspec` solves this problem. `inspect.signature` also handles stripping the `self` argument of methods for us. --- numpy/lib/utils.py | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index f233c7240967..781ef47bc35e 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -9,9 +9,6 @@ from numpy.core import ndarray, ufunc, asarray import numpy as np -# getargspec and formatargspec were removed in Python 3.6 -from numpy.compat import getargspec, formatargspec - __all__ = [ 'issubclass_', 'issubsctype', 'issubdtype', 'deprecate', 'deprecate_with_doc', 'get_include', 'info', 'source', 'who', @@ -552,9 +549,12 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): file=output ) - elif inspect.isfunction(object): + elif inspect.isfunction(object) or inspect.ismethod(object): name = object.__name__ - arguments = formatargspec(*getargspec(object)) + try: + arguments = str(inspect.signature(object)) + except Exception: + arguments = "()" if len(name+arguments) > maxwidth: argstr = _split_line(name, arguments, maxwidth) @@ -566,18 +566,10 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): elif inspect.isclass(object): name = object.__name__ - arguments = "()" try: - if hasattr(object, '__init__'): - arguments = formatargspec( - *getargspec(object.__init__.__func__) - ) - arglist = arguments.split(', ') - if len(arglist) > 1: - arglist[1] = "("+arglist[1] - arguments = ", ".join(arglist[1:]) + arguments = str(inspect.signature(object)) except Exception: - pass + arguments = "()" if len(name+arguments) > maxwidth: argstr = _split_line(name, arguments, maxwidth) @@ -605,26 +597,6 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): ) print(" %s -- %s" % (meth, methstr), file=output) - elif inspect.ismethod(object): - name = object.__name__ - arguments = formatargspec( - *getargspec(object.__func__) - ) - arglist = arguments.split(', ') - if len(arglist) > 1: - arglist[1] = "("+arglist[1] - arguments = ", ".join(arglist[1:]) - else: - arguments = "()" - - if len(name+arguments) > maxwidth: - argstr = _split_line(name, arguments, maxwidth) - else: - argstr = name + arguments - - print(" " + argstr + "\n", file=output) - print(inspect.getdoc(object), file=output) - elif hasattr(object, '__doc__'): print(inspect.getdoc(object), file=output) From 845c48979a4c0b1343037f944a139b88849c6285 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 19 May 2020 13:02:43 +0100 Subject: [PATCH 0068/4003] MAINT: remove workaround for fixed issue take now correctly returns `out`, even on 0d arrays --- numpy/lib/function_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 00f72acba409..95ddcc757a10 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3942,8 +3942,6 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, n = np.array(False, dtype=bool) r = take(ap, indices, axis=0, out=out) - if out is not None: - r = out # workaround for gh-16276 else: # weight the points above and below the indices From e6c4a493d1efb1502af0d85aabb44d7a863d40eb Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Tue, 19 May 2020 15:33:29 +0200 Subject: [PATCH 0069/4003] Move the snippet note --- doc/release/upcoming_changes/15997.improvement.rst | 3 +++ doc/source/release/1.19.0-notes.rst | 1 + doc/source/release/1.20.0-notes.rst | 12 ------------ 3 files changed, 4 insertions(+), 12 deletions(-) create mode 100644 doc/release/upcoming_changes/15997.improvement.rst delete mode 100644 doc/source/release/1.20.0-notes.rst diff --git a/doc/release/upcoming_changes/15997.improvement.rst b/doc/release/upcoming_changes/15997.improvement.rst new file mode 100644 index 000000000000..f211401be937 --- /dev/null +++ b/doc/release/upcoming_changes/15997.improvement.rst @@ -0,0 +1,3 @@ +Improve `array_repr` +-------------------- +Now `array_repr` returns nicely formatted repr for arrays with objects with multi-line reprs. diff --git a/doc/source/release/1.19.0-notes.rst b/doc/source/release/1.19.0-notes.rst index 8b4fb921f2fa..6e7fd69d4167 100644 --- a/doc/source/release/1.19.0-notes.rst +++ b/doc/source/release/1.19.0-notes.rst @@ -3,3 +3,4 @@ ========================== NumPy 1.19.0 Release Notes ========================== + diff --git a/doc/source/release/1.20.0-notes.rst b/doc/source/release/1.20.0-notes.rst deleted file mode 100644 index e6cc2118ba7d..000000000000 --- a/doc/source/release/1.20.0-notes.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. currentmodule:: numpy - -========================== -NumPy 1.20.0 Release Notes -========================== - -Improvements -============ - -Improve `array_repr` ----------------------------------------------- -Now `array_repr` returns nicely formatted repr for arrays with objects with multi-line reprs. From d329a66dbb9710aefd03cce6a8b0f46da51490ca Mon Sep 17 00:00:00 2001 From: Elia Franzella Date: Tue, 19 May 2020 15:39:37 +0200 Subject: [PATCH 0070/4003] MAINT: precompute log(2.0 * M_PI) in `random_loggam' (gh-16237) Most compilers should optimize it, but it doesn't hurt to inline and has a better name now. --- .../random/src/distributions/distributions.c | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/numpy/random/src/distributions/distributions.c b/numpy/random/src/distributions/distributions.c index 586e38aa5bcf..93e0bdc5f6d2 100644 --- a/numpy/random/src/distributions/distributions.c +++ b/numpy/random/src/distributions/distributions.c @@ -22,7 +22,7 @@ static NPY_INLINE float next_float(bitgen_t *bitgen_state) { /* Random generators for external use */ float random_standard_uniform_f(bitgen_t *bitgen_state) { - return next_float(bitgen_state); + return next_float(bitgen_state); } double random_standard_uniform(bitgen_t *bitgen_state) { @@ -342,7 +342,7 @@ uint64_t random_uint(bitgen_t *bitgen_state) { * using logfactorial(k) instead. */ double random_loggam(double x) { - double x0, x2, xp, gl, gl0; + double x0, x2, lg2pi, gl, gl0; RAND_INT_TYPE k, n; static double a[10] = {8.333333333333333e-02, -2.777777777777778e-03, @@ -350,23 +350,25 @@ double random_loggam(double x) { 8.417508417508418e-04, -1.917526917526918e-03, 6.410256410256410e-03, -2.955065359477124e-02, 1.796443723688307e-01, -1.39243221690590e+00}; - x0 = x; - n = 0; + if ((x == 1.0) || (x == 2.0)) { return 0.0; - } else if (x <= 7.0) { + } else if (x < 7.0) { n = (RAND_INT_TYPE)(7 - x); - x0 = x + n; + } else { + n = 0; } - x2 = 1.0 / (x0 * x0); - xp = 2 * M_PI; + x0 = x + n; + x2 = (1.0 / x0) * (1.0 / x0); + /* log(2 * M_PI) */ + lg2pi = 1.8378770664093453e+00; gl0 = a[9]; for (k = 8; k >= 0; k--) { gl0 *= x2; gl0 += a[k]; } - gl = gl0 / x0 + 0.5 * log(xp) + (x0 - 0.5) * log(x0) - x0; - if (x <= 7.0) { + gl = gl0 / x0 + 0.5 * lg2pi + (x0 - 0.5) * log(x0) - x0; + if (x < 7.0) { for (k = 1; k <= n; k++) { gl -= log(x0 - 1.0); x0 -= 1.0; From ba14823f1bf846807985417a69cb05a864683c09 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 19 May 2020 11:03:29 -0500 Subject: [PATCH 0071/4003] BUG: Allow attaching documentation twice in add_docstring This is technically not a bug, but some IDEs and IPython have autoreload magic which can mean that NumPy gets reloaded a second time. This is not safe, but when it happens ignoring that an identical docstring is already attached fixes the issue. --- numpy/core/src/multiarray/compiled_base.c | 11 +++++++--- numpy/tests/test_reloading.py | 26 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 308e72009dcd..5cf18049d734 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -1425,7 +1425,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) #else char *docstr; #endif - static char *msg = "already has a docstring"; + static char *msg = "already has a different docstring"; PyObject *tp_dict = PyArrayDescr_Type.tp_dict; PyObject *myobj; static PyTypeObject *PyMemberDescr_TypePtr = NULL; @@ -1482,7 +1482,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) if (!(doc)) { \ doc = docstr; \ } \ - else { \ + else if (strcmp(doc, docstr) != 0) { \ PyErr_Format(PyExc_RuntimeError, "%s method %s", name, msg); \ return NULL; \ } \ @@ -1507,7 +1507,12 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) PyObject *doc_attr; doc_attr = PyObject_GetAttrString(obj, "__doc__"); - if (doc_attr != NULL && doc_attr != Py_None) { + if (doc_attr != NULL && doc_attr != Py_None && + (PyUnicode_Compare(doc_attr, str) != 0)) { + if (PyErr_Occurred()) { + /* error during PyUnicode_Compare */ + return NULL; + } PyErr_Format(PyExc_RuntimeError, "object %s", msg); return NULL; } diff --git a/numpy/tests/test_reloading.py b/numpy/tests/test_reloading.py index 860832be8767..61ae91b00d55 100644 --- a/numpy/tests/test_reloading.py +++ b/numpy/tests/test_reloading.py @@ -1,8 +1,12 @@ from numpy.testing import assert_raises, assert_, assert_equal from numpy.compat import pickle +import sys +import subprocess +import textwrap from importlib import reload + def test_numpy_reloading(): # gh-7844. Also check that relevant globals retain their identity. import numpy as np @@ -29,3 +33,25 @@ def test_novalue(): assert_equal(repr(np._NoValue), '') assert_(pickle.loads(pickle.dumps(np._NoValue, protocol=proto)) is np._NoValue) + + +def test_full_reimport(): + """At the time of writing this, it is *not* truly supported, but + apparently enough users rely on it, for it to be an annoying change + when it started failing previously. + """ + # Test within a new process, to ensure that we do not mess with the + # global state during the test run (could lead to cryptic test failures). + # This is generally unsafe, especially, since we also reload the C-modules. + code = textwrap.dedent(r""" + import sys + import numpy as np + + for k in list(sys.modules.keys()): + if "numpy" in k: + del sys.modules[k] + + import numpy as np + """) + p = subprocess.run([sys.executable, '-c', code]) + assert p.returncode == 0 From c50a305534f11c6600ad7728ae91208b79ec3593 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 19 May 2020 11:05:16 -0500 Subject: [PATCH 0072/4003] DOC: Fix generic scalar docstrings to point to ndarray Previously some of these were not set, because they were already set on the C-level. Remove the duplicate message and replace it instead with a forward to the ``ndarray`` attribute/method. These docstrings are inherited by the actual scalars and thus the "virtual class" note was misleading. --- numpy/core/_add_newdocs.py | 678 ++++---------------- numpy/core/src/multiarray/scalartypes.c.src | 56 +- 2 files changed, 124 insertions(+), 610 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index f43b77c44c08..20ad39b05ed6 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -5901,29 +5901,21 @@ # Attributes -add_newdoc('numpy.core.numerictypes', 'generic', ('T', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class so as to - provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) +def refer_to_array_attribute(attr, method=True): + docstring = """ + Scalar {} identical to the corresponding array attribute. -add_newdoc('numpy.core.numerictypes', 'generic', ('base', + Please see `ndarray.{}`. """ - Not implemented (virtual attribute) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class so as to - a uniform API. + return attr, docstring.format("method" if method else "attribute", attr) - See also the corresponding attribute of the derived class of interest. - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('T', method=False)) + +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('base', method=False)) add_newdoc('numpy.core.numerictypes', 'generic', ('data', """Pointer to start of data.""")) @@ -5963,305 +5955,80 @@ # Methods -add_newdoc('numpy.core.numerictypes', 'generic', ('all', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('any', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('argmax', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('argmin', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('argsort', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('astype', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('byteswap', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class so as to - provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('choose', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('clip', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('compress', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('conjugate', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('copy', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('cumprod', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('cumsum', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('diagonal', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('dump', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('dumps', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('fill', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('flatten', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('all')) -add_newdoc('numpy.core.numerictypes', 'generic', ('getfield', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('any')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('argmax')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('argmin')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('argsort')) -add_newdoc('numpy.core.numerictypes', 'generic', ('item', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('astype')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('byteswap')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('choose')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('clip')) -add_newdoc('numpy.core.numerictypes', 'generic', ('itemset', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('compress')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('conjugate')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('copy')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('cumprod')) -add_newdoc('numpy.core.numerictypes', 'generic', ('max', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('cumsum')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('diagonal')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('dump')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('dumps')) -add_newdoc('numpy.core.numerictypes', 'generic', ('mean', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('fill')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('flatten')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('getfield')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('item')) -add_newdoc('numpy.core.numerictypes', 'generic', ('min', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('itemset')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('max')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('mean')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('min')) add_newdoc('numpy.core.numerictypes', 'generic', ('newbyteorder', """ @@ -6296,305 +6063,80 @@ """)) -add_newdoc('numpy.core.numerictypes', 'generic', ('nonzero', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('prod', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('ptp', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('put', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('ravel', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('repeat', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('reshape', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('resize', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('round', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('searchsorted', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('setfield', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('setflags', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class so as to - provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('sort', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('squeeze', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('std', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('sum', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('swapaxes', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('take', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) - -add_newdoc('numpy.core.numerictypes', 'generic', ('tofile', - """ - Not implemented (virtual attribute) - - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. - - See also the corresponding attribute of the derived class of interest. - - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('nonzero')) -add_newdoc('numpy.core.numerictypes', 'generic', ('tolist', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('prod')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('ptp')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('put')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('ravel')) -add_newdoc('numpy.core.numerictypes', 'generic', ('tostring', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('repeat')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('reshape')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('resize')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('round')) -add_newdoc('numpy.core.numerictypes', 'generic', ('trace', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('searchsorted')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('setfield')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('setflags')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('sort')) -add_newdoc('numpy.core.numerictypes', 'generic', ('transpose', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('squeeze')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('std')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('sum')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('swapaxes')) -add_newdoc('numpy.core.numerictypes', 'generic', ('var', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('take')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('tofile')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('tolist')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('tostring')) -add_newdoc('numpy.core.numerictypes', 'generic', ('view', - """ - Not implemented (virtual attribute) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('trace')) - Class generic exists solely to derive numpy scalars from, and possesses, - albeit unimplemented, all the attributes of the ndarray class - so as to provide a uniform API. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('transpose')) - See also the corresponding attribute of the derived class of interest. +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('var')) - """)) +add_newdoc('numpy.core.numerictypes', 'generic', + refer_to_array_attribute('view')) ############################################################################## diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 2f1767391125..f13f50759fb6 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -1333,74 +1333,46 @@ gentype_transpose_get(PyObject *self) static PyGetSetDef gentype_getsets[] = { {"ndim", (getter)gentype_ndim_get, - (setter) 0, - "number of array dimensions", - NULL}, + (setter) 0, NULL, NULL}, {"flags", (getter)gentype_flags_get, - (setter)0, - "integer value of flags", - NULL}, + (setter)0, NULL, NULL}, {"shape", (getter)gentype_shape_get, - (setter)0, - "tuple of array dimensions", - NULL}, + (setter)0, NULL, NULL}, {"strides", (getter)gentype_shape_get, - (setter) 0, - "tuple of bytes steps in each dimension", - NULL}, + (setter) 0, NULL, NULL}, {"data", (getter)gentype_data_get, - (setter) 0, - "pointer to start of data", - NULL}, + (setter) 0, NULL, NULL}, {"itemsize", (getter)gentype_itemsize_get, - (setter)0, - "length of one element in bytes", - NULL}, + (setter)0, NULL, NULL}, {"size", (getter)gentype_size_get, - (setter)0, - "number of elements in the gentype", - NULL}, + (setter)0, NULL, NULL}, {"nbytes", (getter)gentype_itemsize_get, - (setter)0, - "length of item in bytes", - NULL}, + (setter)0, NULL, NULL}, {"base", (getter)gentype_base_get, - (setter)0, - "base object", - NULL}, + (setter)0, NULL, NULL}, {"dtype", (getter)gentype_typedescr_get, - NULL, - "get array data-descriptor", - NULL}, + NULL, NULL, NULL}, {"real", (getter)gentype_real_get, - (setter)0, - "real part of scalar", - NULL}, + (setter)0, NULL, NULL}, {"imag", (getter)gentype_imag_get, - (setter)0, - "imaginary part of scalar", - NULL}, + (setter)0, NULL, NULL}, {"flat", (getter)gentype_flat_get, - (setter)0, - "a 1-d view of scalar", - NULL}, + (setter)0, NULL, NULL}, {"T", (getter)gentype_transpose_get, - (setter)0, - "transpose", - NULL}, + (setter)0, NULL, NULL}, {"__array_interface__", (getter)gentype_interface_get, NULL, From 043799295f82b01235479d5b323de4598dca48a0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 19 May 2020 14:38:05 -0700 Subject: [PATCH 0073/4003] DOC: Fix documentation rendering, the => seem to be extra and rst seem to not interprete backtick with leading space as closing a directive, the the full line on text was put in a html pre-tag when building the docs. see current https://numpy.org/doc/stable/reference/generated/numpy.packbits.html --- numpy/core/multiarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 5ae6a42720c1..2cc2a8e710c1 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -1139,7 +1139,7 @@ def packbits(a, axis=None, bitorder='big'): ``None`` implies packing the flattened array. bitorder : {'big', 'little'}, optional The order of the input bits. 'big' will mimic bin(val), - ``[0, 0, 0, 0, 0, 0, 1, 1] => 3 = 0b00000011 => ``, 'little' will + ``[0, 0, 0, 0, 0, 0, 1, 1] => 3 = 0b00000011``, 'little' will reverse the order so ``[1, 1, 0, 0, 0, 0, 0, 0] => 3``. Defaults to 'big'. From f6196441b7240c0fa862790efdfb1a62bf557744 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Wed, 20 May 2020 13:20:40 +0300 Subject: [PATCH 0074/4003] BUG: relpath fails for different drives on windows (#16308) * BUG: relpath fails for different drives on windows * ENH: always use abspath --- numpy/distutils/command/build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 078b8fb5953e..d53285c92b05 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -520,7 +520,7 @@ def _process_unlinkable_fobjects(self, objects, libraries, # Wrap unlinkable objects to a linkable one if unlinkable_fobjects: - fobjects = [os.path.relpath(obj) for obj in unlinkable_fobjects] + fobjects = [os.path.abspath(obj) for obj in unlinkable_fobjects] wrapped = fcompiler.wrap_unlinkable_objects( fobjects, output_dir=self.build_temp, extra_dll_dir=self.extra_dll_dir) From b0e385f2df3b817c58a0ec043854e91151e984a6 Mon Sep 17 00:00:00 2001 From: Pan Jan Date: Wed, 20 May 2020 15:36:00 +0200 Subject: [PATCH 0075/4003] Improve the snippet --- doc/release/upcoming_changes/15997.improvement.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/release/upcoming_changes/15997.improvement.rst b/doc/release/upcoming_changes/15997.improvement.rst index f211401be937..7f150c3db0df 100644 --- a/doc/release/upcoming_changes/15997.improvement.rst +++ b/doc/release/upcoming_changes/15997.improvement.rst @@ -1,3 +1,4 @@ -Improve `array_repr` --------------------- -Now `array_repr` returns nicely formatted repr for arrays with objects with multi-line reprs. +Improve `repr(arr)` +------------------- +Now `repr(arr)` returns nicely formatted repr for arrays with objects with +multi-line reprs. From d758e58b76f7168dd59c449c739f4260edad25f2 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 15 May 2020 15:42:27 -0500 Subject: [PATCH 0076/4003] TST: Add a test for np.add_docstring Its not quite the right file, but close to newdoc seemed sensible and we do not have a "right" file right now... --- numpy/lib/tests/test_function_base.py | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index b4e92827313a..008ea0759db9 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -3350,12 +3350,50 @@ class TestAdd_newdoc: @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_add_doc(self): - # test np.add_newdoc + # test that np.add_newdoc did attach a docstring successfully: tgt = "Current flat index into the array." assert_equal(np.core.flatiter.index.__doc__[:len(tgt)], tgt) assert_(len(np.core.ufunc.identity.__doc__) > 300) assert_(len(np.lib.index_tricks.mgrid.__doc__) > 300) + @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + def test_errors_are_ignored(self): + prev_doc = np.core.flatiter.index.__doc__ + # nothing changed, but error ignored, this should probably + # give a warning (or even error) in the future. + np.add_newdoc("numpy.core", "flatiter", ("index", "bad docstring")) + assert prev_doc == np.core.flatiter.index.__doc__ + + +class TestAddDocstring(): + # Test should possibly be moved, but it also fits to be close to + # the newdoc tests... + @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + @pytest.mark.skipif(IS_PYPY, reason="PyPy does not modify tp_doc") + def test_add_same_docstring(self): + # test for attributes (which are C-level defined) + np.add_docstring(np.ndarray.flat, np.ndarray.flat.__doc__) + # And typical functions: + def func(): + """docstring""" + return + + np.add_docstring(func, func.__doc__) + + @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + def test_different_docstring_fails(self): + # test for attributes (which are C-level defined) + with assert_raises(RuntimeError): + np.add_docstring(np.ndarray.flat, "different docstring") + # And typical functions: + def func(): + """docstring""" + return + + with assert_raises(RuntimeError): + np.add_docstring(func, "different docstring") + + class TestSortComplex: @pytest.mark.parametrize("type_in, type_out", [ From ffdce8b4ddf725d9f4329a745d6bfad9b5716774 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 19 May 2020 11:08:15 -0500 Subject: [PATCH 0077/4003] MAINT: Simplify logic in add_docstring The old pointers are now provided by the C-API and the macros seemed a bit confusing since ``new`` appears from nowhere being defined in the macro. --- numpy/core/src/multiarray/compiled_base.c | 69 ++++++----------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 5cf18049d734..b63bca368661 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -1426,46 +1426,12 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) char *docstr; #endif static char *msg = "already has a different docstring"; - PyObject *tp_dict = PyArrayDescr_Type.tp_dict; - PyObject *myobj; - static PyTypeObject *PyMemberDescr_TypePtr = NULL; - static PyTypeObject *PyGetSetDescr_TypePtr = NULL; - static PyTypeObject *PyMethodDescr_TypePtr = NULL; /* Don't add docstrings */ if (Py_OptimizeFlag > 1) { Py_RETURN_NONE; } - if (PyGetSetDescr_TypePtr == NULL) { - /* Get "subdescr" */ - myobj = _PyDict_GetItemStringWithError(tp_dict, "fields"); - if (myobj == NULL && PyErr_Occurred()) { - return NULL; - } - if (myobj != NULL) { - PyGetSetDescr_TypePtr = Py_TYPE(myobj); - } - } - if (PyMemberDescr_TypePtr == NULL) { - myobj = _PyDict_GetItemStringWithError(tp_dict, "alignment"); - if (myobj == NULL && PyErr_Occurred()) { - return NULL; - } - if (myobj != NULL) { - PyMemberDescr_TypePtr = Py_TYPE(myobj); - } - } - if (PyMethodDescr_TypePtr == NULL) { - myobj = _PyDict_GetItemStringWithError(tp_dict, "newbyteorder"); - if (myobj == NULL && PyErr_Occurred()) { - return NULL; - } - if (myobj != NULL) { - PyMethodDescr_TypePtr = Py_TYPE(myobj); - } - } - if (!PyArg_ParseTuple(args, "OO!:add_docstring", &obj, &PyUnicode_Type, &str)) { return NULL; } @@ -1475,33 +1441,34 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) return NULL; } -#define _TESTDOC1(typebase) (Py_TYPE(obj) == &Py##typebase##_Type) -#define _TESTDOC2(typebase) (Py_TYPE(obj) == Py##typebase##_TypePtr) -#define _ADDDOC(typebase, doc, name) do { \ - Py##typebase##Object *new = (Py##typebase##Object *)obj; \ +#define _ADDDOC(doc, name) \ if (!(doc)) { \ doc = docstr; \ } \ else if (strcmp(doc, docstr) != 0) { \ PyErr_Format(PyExc_RuntimeError, "%s method %s", name, msg); \ return NULL; \ - } \ - } while (0) + } - if (_TESTDOC1(CFunction)) { - _ADDDOC(CFunction, new->m_ml->ml_doc, new->m_ml->ml_name); + if (Py_TYPE(obj) == &PyCFunction_Type) { + PyCFunctionObject *new = (PyCFunctionObject *)obj; + _ADDDOC(new->m_ml->ml_doc, new->m_ml->ml_name); } - else if (_TESTDOC1(Type)) { - _ADDDOC(Type, new->tp_doc, new->tp_name); + else if (Py_TYPE(obj) == &PyType_Type) { + PyTypeObject *new = (PyTypeObject *)obj; + _ADDDOC(new->tp_doc, new->tp_name); } - else if (_TESTDOC2(MemberDescr)) { - _ADDDOC(MemberDescr, new->d_member->doc, new->d_member->name); + else if (Py_TYPE(obj) == &PyMemberDescr_Type) { + PyMemberDescrObject *new = (PyMemberDescrObject *)obj; + _ADDDOC(new->d_member->doc, new->d_member->name); } - else if (_TESTDOC2(GetSetDescr)) { - _ADDDOC(GetSetDescr, new->d_getset->doc, new->d_getset->name); + else if (Py_TYPE(obj) == &PyGetSetDescr_Type) { + PyGetSetDescrObject *new = (PyGetSetDescrObject *)obj; + _ADDDOC(new->d_getset->doc, new->d_getset->name); } - else if (_TESTDOC2(MethodDescr)) { - _ADDDOC(MethodDescr, new->d_method->ml_doc, new->d_method->ml_name); + else if (Py_TYPE(obj) == &PyMethodDescr_Type) { + PyMethodDescrObject *new = (PyMethodDescrObject *)obj; + _ADDDOC(new->d_method->ml_doc, new->d_method->ml_name); } else { PyObject *doc_attr; @@ -1526,8 +1493,6 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) Py_RETURN_NONE; } -#undef _TESTDOC1 -#undef _TESTDOC2 #undef _ADDDOC Py_INCREF(str); From dcf1614dfc11f9004e408ed8c833980c951c541e Mon Sep 17 00:00:00 2001 From: "E. M. Bray" Date: Wed, 20 May 2020 16:00:20 +0200 Subject: [PATCH 0078/4003] BLD: Avoid "visibility attribute not supported" warning (gh-16288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I found that when building the latest master branch on Cygwin, while testing #16246, that thousands of warnings were generated at build time like: numpy/core/src/npysort/binsearch.c.src: In function ‘binsearch_left_bool’: numpy/core/src/npysort/binsearch.c.src:82:1: warning: visibility attribute not supported in this configuration; ignored [-Wattributes] Granted this is just a warning, so I don't think it's a serious issue. It seems the test that was supposed to check for __attribute__ support was not working as expected. The #pragmas only take effect if I provide a function body--they are ignored for bare declarations. I don't know if that's by intent, or if it's a GCC issue. For reference: $ gcc --version gcc (GCC) 7.4.0 --- numpy/distutils/command/autodist.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numpy/distutils/command/autodist.py b/numpy/distutils/command/autodist.py index 1475a5e241ee..8f6436004a0a 100644 --- a/numpy/distutils/command/autodist.py +++ b/numpy/distutils/command/autodist.py @@ -69,7 +69,10 @@ def check_gcc_function_attribute(cmd, attribute, name): #pragma GCC diagnostic error "-Wattributes" #pragma clang diagnostic error "-Wattributes" - int %s %s(void*); + int %s %s(void* unused) + { + return 0; + } int main() From e29a94a9becbdeb0b8229c5395a70b2cbd971cf8 Mon Sep 17 00:00:00 2001 From: panpiort8 Date: Wed, 20 May 2020 16:33:47 +0200 Subject: [PATCH 0079/4003] Update doc/release/upcoming_changes/15997.improvement.rst Co-authored-by: Eric Wieser --- .../upcoming_changes/15997.improvement.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/release/upcoming_changes/15997.improvement.rst b/doc/release/upcoming_changes/15997.improvement.rst index 7f150c3db0df..9b5feacb83a3 100644 --- a/doc/release/upcoming_changes/15997.improvement.rst +++ b/doc/release/upcoming_changes/15997.improvement.rst @@ -1,4 +1,12 @@ -Improve `repr(arr)` -------------------- -Now `repr(arr)` returns nicely formatted repr for arrays with objects with -multi-line reprs. +Object arrays containing multi-line objects have a more readable ``repr`` +------------------------------------------------------------------------- +If elements of an object array have a ``repr`` containing new lines, then the +wrapped lines will be aligned by column. Notably, this improves the ``repr`` of +nested arrays:: + + >>> np.array([np.eye(2), np.eye(3)], dtype=object) + array([array([[1., 0.], + [0., 1.]]), + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]])], dtype=object) From 18eefadf0ead841d21279f5a533165429531f3b3 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 20 May 2020 09:48:06 -0700 Subject: [PATCH 0080/4003] MAINT: Bumpy numpydoc version. Fixes problem with citation (reference) anchors in rendered docs. Actual fix was in numpy/numpydoc@7c42883, but this bumps to latest (unreleased) numpydoc version. --- doc/sphinxext | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinxext b/doc/sphinxext index a482f66913c1..b4c5fd17e2b8 160000 --- a/doc/sphinxext +++ b/doc/sphinxext @@ -1 +1 @@ -Subproject commit a482f66913c1079d7439770f0119b55376bb1b81 +Subproject commit b4c5fd17e2b85c2416a5e586933eee353b58bf7c From a77c3f646e25ff806daeb40d2ee5b9ad0ed74588 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 20 May 2020 14:57:26 -0500 Subject: [PATCH 0081/4003] MAINT: Stop Using PyEval_Call* and simplify some uses Python 3.9 started to deprecate these calls and the PyObject_Call* variants are equivalent or nicer in any case. --- numpy/core/src/multiarray/strfuncs.c | 12 ++++-------- numpy/core/src/umath/loops.c.src | 11 ++--------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/strfuncs.c b/numpy/core/src/multiarray/strfuncs.c index b570aec08b73..363cbdba2d01 100644 --- a/numpy/core/src/multiarray/strfuncs.c +++ b/numpy/core/src/multiarray/strfuncs.c @@ -168,15 +168,13 @@ array_repr_builtin(PyArrayObject *self, int repr) NPY_NO_EXPORT PyObject * array_repr(PyArrayObject *self) { - PyObject *s, *arglist; + PyObject *s; if (PyArray_ReprFunction == NULL) { s = array_repr_builtin(self, 1); } else { - arglist = Py_BuildValue("(O)", self); - s = PyEval_CallObject(PyArray_ReprFunction, arglist); - Py_DECREF(arglist); + s = PyObject_CallFunctionObjArgs(PyArray_ReprFunction, self, NULL); } return s; } @@ -185,15 +183,13 @@ array_repr(PyArrayObject *self) NPY_NO_EXPORT PyObject * array_str(PyArrayObject *self) { - PyObject *s, *arglist; + PyObject *s; if (PyArray_StrFunction == NULL) { s = array_repr_builtin(self, 0); } else { - arglist = Py_BuildValue("(O)", self); - s = PyEval_CallObject(PyArray_StrFunction, arglist); - Py_DECREF(arglist); + s = PyObject_CallFunctionObjArgs(PyArray_StrFunction, self, NULL); } return s; } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index a5c663a47ba6..a59a9acf5aef 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -225,10 +225,6 @@ NPY_NO_EXPORT void PyUFunc_O_O_method(char **args, npy_intp const *dimensions, npy_intp const *steps, void *func) { char *meth = (char *)func; - PyObject *tup = PyTuple_New(0); - if (tup == NULL) { - return; - } UNARY_LOOP { PyObject *in1 = *(PyObject **)ip1; PyObject **out = (PyObject **)op1; @@ -247,20 +243,17 @@ PyUFunc_O_O_method(char **args, npy_intp const *dimensions, npy_intp const *step "type %s which has no callable %s method", i, type->tp_name, meth); npy_PyErr_ChainExceptionsCause(exc, val, tb); - Py_DECREF(tup); Py_XDECREF(func); return; } - ret = PyObject_Call(func, tup, NULL); + ret = PyObject_CallObject(func, NULL); Py_DECREF(func); if (ret == NULL) { - Py_DECREF(tup); return; } Py_XDECREF(*out); *out = ret; } - Py_DECREF(tup); } /*UFUNC_API*/ @@ -359,7 +352,7 @@ PyUFunc_On_Om(char **args, npy_intp const *dimensions, npy_intp const *steps, vo PyTuple_SET_ITEM(arglist, j, in); Py_INCREF(in); } - result = PyEval_CallObject(tocall, arglist); + result = PyObject_CallObject(tocall, arglist); Py_DECREF(arglist); if (result == NULL) { return; From 3e9934edd1c15e943861ef3332d7ba513541f2d4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 20 May 2020 14:59:42 -0500 Subject: [PATCH 0082/4003] TST: Add 3.9-dev to the travis matrix Co-authored-by: Charles Harris --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d51a1d223bef..e019495fb852 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,7 @@ jobs: - stage: Comprehensive tests python: 3.6 - python: 3.7 + - python: 3.9-dev - python: 3.6 env: USE_DEBUG=1 From 1d57e850d5895be4bc928459059c241818f84c2c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 19 May 2020 17:24:14 -0500 Subject: [PATCH 0083/4003] DOC: Fix troubleshooting code snippet when env vars are empty The previous snippet worked fine, but not when the environment variables were completly empty. --- doc/source/user/troubleshooting-importerror.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/source/user/troubleshooting-importerror.rst b/doc/source/user/troubleshooting-importerror.rst index acbcd3ed8edb..7d4846f7794f 100644 --- a/doc/source/user/troubleshooting-importerror.rst +++ b/doc/source/user/troubleshooting-importerror.rst @@ -133,10 +133,8 @@ your system. If you can open a correct python shell, you can also run the following in python:: import os - PYTHONPATH = os.environ['PYTHONPATH'].split(os.pathsep) - print("The PYTHONPATH is:", PYTHONPATH) - PATH = os.environ['PATH'].split(os.pathsep) - print("The PATH is:", PATH) + print("PYTHONPATH:", os.environ.get('PYTHONPATH')) + print("PATH:", os.environ.get('PATH')) This may mainly help you if you are not running the python and/or NumPy version you are expecting to run. From 41e254f96e8d5bd558d2edbf8b198eb4143e8b74 Mon Sep 17 00:00:00 2001 From: Tina Oberoi <55803680+tinaoberoi@users.noreply.github.com> Date: Thu, 21 May 2020 15:01:47 +0530 Subject: [PATCH 0084/4003] DOC: Correct documentation of ``__array__`` when used as output array. (#16130) --- doc/source/reference/arrays.classes.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 6b6f366dfcd9..6c8793342093 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -326,9 +326,8 @@ NumPy provides several hooks that classes can customize: If a class (ndarray subclass or not) having the :func:`__array__` method is used as the output object of an :ref:`ufunc - `, results will be written to the object - returned by :func:`__array__`. Similar conversion is done on - input arrays. + `, results will *not* be written to the object + returned by :func:`__array__`. This practice will return ``TypeError``. Matrix objects From 40d95016cd4322530e0aa28a9b4ae86a95eb8d02 Mon Sep 17 00:00:00 2001 From: skywalker <65246829+LSchroefl@users.noreply.github.com> Date: Thu, 21 May 2020 21:53:49 +0200 Subject: [PATCH 0085/4003] DOC: link np.interp to SciPy's multidimensional interpolation functions (#14154) --- numpy/lib/function_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 2d1adc362f61..bb6a84795b2f 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1332,6 +1332,10 @@ def interp(x, xp, fp, left=None, right=None, period=None): If `xp` or `fp` are not 1-D sequences If `period == 0` + See also + -------- + scipy.interpolate + Notes ----- The x-coordinate sequence is expected to be increasing, but this is not From 66c756ce333dc40120d4dd77b14517c6819cd410 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 21 May 2020 13:29:34 -0700 Subject: [PATCH 0086/4003] BUG: Don't segfault on bad __len__ when assigning. (gh-16327) When __getitem__ fails, assignment falls back on __iter__, which may not have the same length as __len__, resulting in a segfault. See gh-7264. * BUG: Don't rely on __len__ for safe iteration. Skip __len__ checking entirely, since this has no guarantees of being correct. Instead, first convert to PySequence_Fast and use that length. Also fixes a refleak when creating a tmp array fails. See gh-7264. * TST: Update test for related edge-case. The previous fix more gracefully handles this edge case by skipping the __len__ check. Rather than raising with an unhelpful error message, just create the array with whatever elements C() actually yields. See gh-7264. --- numpy/core/src/multiarray/ctors.c | 26 ++++++++------------------ numpy/core/tests/test_multiarray.py | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 9283eefce872..502ab0ea916e 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -483,6 +483,8 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, /* INCREF on entry DECREF on exit */ Py_INCREF(s); + PyObject *seq = NULL; + if (PyArray_Check(s)) { if (!(PyArray_CheckExact(s))) { /* @@ -529,10 +531,11 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, return 0; } - slen = PySequence_Length(s); - if (slen < 0) { + seq = PySequence_Fast(s, "Could not convert object to sequence"); + if (seq == NULL) { goto fail; } + slen = PySequence_Fast_GET_SIZE(seq); /* * Either the dimensions match, or the sequence has length 1 and can @@ -547,14 +550,9 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, /* Broadcast the one element from the sequence to all the outputs */ if (slen == 1) { - PyObject *o; + PyObject *o = PySequence_Fast_GET_ITEM(seq, 0); npy_intp alen = PyArray_DIM(a, dim); - o = PySequence_GetItem(s, 0); - if (o == NULL) { - goto fail; - } - for (i = 0; i < alen; i++) { if ((PyArray_NDIM(a) - dim) > 1) { PyArrayObject * tmp = @@ -571,26 +569,18 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, res = PyArray_SETITEM(dst, b, o); } if (res < 0) { - Py_DECREF(o); goto fail; } } - Py_DECREF(o); } /* Copy element by element */ else { - PyObject * seq; - seq = PySequence_Fast(s, "Could not convert object to sequence"); - if (seq == NULL) { - goto fail; - } for (i = 0; i < slen; i++) { PyObject * o = PySequence_Fast_GET_ITEM(seq, i); if ((PyArray_NDIM(a) - dim) > 1) { PyArrayObject * tmp = (PyArrayObject *)array_item_asarray(dst, i); if (tmp == NULL) { - Py_DECREF(seq); goto fail; } @@ -602,17 +592,17 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, res = PyArray_SETITEM(dst, b, o); } if (res < 0) { - Py_DECREF(seq); goto fail; } } - Py_DECREF(seq); } + Py_DECREF(seq); Py_DECREF(s); return 0; fail: + Py_XDECREF(seq); Py_DECREF(s); return res; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index a698370b6950..fc2ab94e6d86 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -988,7 +988,22 @@ def __getitem__(self, i): def __len__(self): return 42 - assert_raises(ValueError, np.array, C()) # segfault? + a = np.array(C()) # segfault? + assert_equal(len(a), 0) + + def test_false_len_iterable(self): + # Special case where a bad __getitem__ makes us fall back on __iter__: + class C: + def __getitem__(self, x): + raise Exception + def __iter__(self): + return iter(()) + def __len__(self): + return 2 + + a = np.empty(2) + with assert_raises(ValueError): + a[:] = C() # Segfault! def test_failed_len_sequence(self): # gh-7393 @@ -1804,7 +1819,7 @@ def test_sort_object(self): c = b.copy() c.sort(kind=kind) assert_equal(c, a, msg) - + def test_sort_structured(self): # test record array sorts. dt = np.dtype([('f', float), ('i', int)]) From 6dfce7f498a9ee7f4cb396105270793ddc828e80 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Thu, 21 May 2020 16:34:15 -0400 Subject: [PATCH 0087/4003] DOC: Fix spelling typo - homogenous to homogeneous. (#16324) --- doc/neps/nep-0022-ndarray-duck-typing-overview.rst | 2 +- doc/source/reference/arrays.interface.rst | 2 +- doc/source/reference/arrays.rst | 2 +- doc/source/user/absolute_beginners.rst | 4 ++-- numpy/core/tests/test_multiarray.py | 2 +- numpy/doc/glossary.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/neps/nep-0022-ndarray-duck-typing-overview.rst b/doc/neps/nep-0022-ndarray-duck-typing-overview.rst index 077166453b60..9ece915a72e8 100644 --- a/doc/neps/nep-0022-ndarray-duck-typing-overview.rst +++ b/doc/neps/nep-0022-ndarray-duck-typing-overview.rst @@ -23,7 +23,7 @@ Detailed description -------------------- Traditionally, NumPy’s ``ndarray`` objects have provided two things: a -high level API for expression operations on homogenously-typed, +high level API for expression operations on homogeneously-typed, arbitrary-dimensional, array-structured data, and a concrete implementation of the API based on strided in-RAM storage. The API is powerful, fairly general, and used ubiquitously across the scientific diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index f36a083aa5f9..4e95535c0605 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -71,7 +71,7 @@ This approach to the interface consists of the object having an **typestr** (required) - A string providing the basic type of the homogenous array The + A string providing the basic type of the homogeneous array The basic string format consists of 3 parts: a character describing the byteorder of the data (``<``: little-endian, ``>``: big-endian, ``|``: not-relevant), a character code giving the diff --git a/doc/source/reference/arrays.rst b/doc/source/reference/arrays.rst index faa91a389562..497dd9cd6f51 100644 --- a/doc/source/reference/arrays.rst +++ b/doc/source/reference/arrays.rst @@ -11,7 +11,7 @@ NumPy provides an N-dimensional array type, the :ref:`ndarray type. The items can be :ref:`indexed ` using for example N integers. -All ndarrays are :term:`homogenous`: every item takes up the same size +All ndarrays are :term:`homogeneous`: every item takes up the same size block of memory, and all blocks are interpreted in exactly the same way. How each item in the array is to be interpreted is specified by a separate :ref:`data-type object `, one of which is associated diff --git a/doc/source/user/absolute_beginners.rst b/doc/source/user/absolute_beginners.rst index ad2cd2d80524..bd44b70daa67 100644 --- a/doc/source/user/absolute_beginners.rst +++ b/doc/source/user/absolute_beginners.rst @@ -100,8 +100,8 @@ What’s the difference between a Python list and a NumPy array? NumPy gives you an enormous range of fast and efficient ways of creating arrays and manipulating numerical data inside them. While a Python list can contain different data types within a single list, all of the elements in a NumPy array -should be homogenous. The mathematical operations that are meant to be performed -on arrays would be extremely inefficient if the arrays weren't homogenous. +should be homogeneous. The mathematical operations that are meant to be performed +on arrays would be extremely inefficient if the arrays weren't homogeneous. **Why use NumPy?** diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index a698370b6950..9b21448f55ea 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -929,7 +929,7 @@ def test_empty_unicode(self): d = np.empty(i, dtype='U') str(d) - def test_sequence_non_homogenous(self): + def test_sequence_non_homogeneous(self): assert_equal(np.array([4, 2**80]).dtype, object) assert_equal(np.array([4, 2**80, 4]).dtype, object) assert_equal(np.array([2**80, 4]).dtype, object) diff --git a/numpy/doc/glossary.py b/numpy/doc/glossary.py index 16a3b92417cb..31130559b744 100644 --- a/numpy/doc/glossary.py +++ b/numpy/doc/glossary.py @@ -169,7 +169,7 @@ Collapsed to a one-dimensional array. See `numpy.ndarray.flatten` for details. - homogenous + homogeneous Describes a block of memory comprised of blocks, each block comprised of items and of the same size, and blocks are interpreted in exactly the same way. In the simplest case each block contains a single item, for From 15ffea3966bab19d2390971f7254f32f150ae93f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 16 May 2020 14:22:00 -0700 Subject: [PATCH 0088/4003] ENH: include dt64/td64 isinstance checks in __init__.pxd --- numpy/__init__.pxd | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index d5c50d9bf219..721c668fa042 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -26,6 +26,7 @@ cimport libc.stdio as stdio cdef extern from "Python.h": ctypedef int Py_intptr_t + bint PyObject_TypeCheck(object obj, PyTypeObject* type) nogil cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -688,6 +689,22 @@ cdef extern from "numpy/arrayobject.h": int PyArray_SetBaseObject(ndarray, base) # NOTE: steals a reference to base! Use "set_array_base()" instead. +cdef extern from "numpy/ndarrayobject.h": + PyTypeObject PyTimedeltaArrType_Type + PyTypeObject PyDatetimeArrType_Type + +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + +cdef extern from "numpy/arrayscalars.h": + ctypedef struct PyDatetimeScalarObject: + # PyObject_HEAD + npy_datetime obval + PyArray_DatetimeMetaData obmeta + + # Typedefs that matches the runtime dtype objects in # the numpy module. @@ -976,3 +993,57 @@ cdef extern from *: """ /* NumPy API declarations from "numpy/__init__.pxd" */ """ + + +cdef inline bint is_timedelta64_object(object obj) nogil: + """ + Cython equivalent of `isinstance(obj, np.timedelta64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) + + +cdef inline bint is_datetime64_object(object obj) nogil: + """ + Cython equivalent of `isinstance(obj, np.datetime64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) + + +cdef inline npy_datetime get_datetime64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy datetime64 object + + Note that to interpret this as a datetime, the corresponding unit is + also needed. That can be found using `get_datetime64_unit`. + """ + return (obj).obval + + +cdef inline npy_timedelta get_timedelta64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy timedelta64 object + """ + return (obj).obval + + +cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) nogil: + """ + returns the unit part of the dtype for a numpy datetime64 object. + """ + return (obj).obmeta.base From 76ead088453510c9e7a339b899b1dbd2ae3ed4d4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 18 May 2020 12:53:59 -0700 Subject: [PATCH 0089/4003] Re-order declarations --- numpy/__init__.pxd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index 721c668fa042..f34d0f9d71c7 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -693,17 +693,17 @@ cdef extern from "numpy/ndarrayobject.h": PyTypeObject PyTimedeltaArrType_Type PyTypeObject PyDatetimeArrType_Type -cdef extern from "numpy/ndarraytypes.h": - ctypedef struct PyArray_DatetimeMetaData: - NPY_DATETIMEUNIT base - int64_t num - cdef extern from "numpy/arrayscalars.h": ctypedef struct PyDatetimeScalarObject: # PyObject_HEAD npy_datetime obval PyArray_DatetimeMetaData obmeta +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + # Typedefs that matches the runtime dtype objects in # the numpy module. From 82684487cacc3ce76aba9fa1a0f1f1a8bb17fa21 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 18 May 2020 13:09:45 -0700 Subject: [PATCH 0090/4003] Missing declarations --- numpy/__init__.pxd | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index f34d0f9d71c7..bd7f169f5363 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -692,6 +692,8 @@ cdef extern from "numpy/arrayobject.h": cdef extern from "numpy/ndarrayobject.h": PyTypeObject PyTimedeltaArrType_Type PyTypeObject PyDatetimeArrType_Type + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime cdef extern from "numpy/arrayscalars.h": ctypedef struct PyDatetimeScalarObject: @@ -699,11 +701,32 @@ cdef extern from "numpy/arrayscalars.h": npy_datetime obval PyArray_DatetimeMetaData obmeta + ctypedef struct PyTimedeltaScalarObject: + # PyObject_HEAD + npy_timedelta obval + PyArray_DatetimeMetaData obmeta + cdef extern from "numpy/ndarraytypes.h": ctypedef struct PyArray_DatetimeMetaData: NPY_DATETIMEUNIT base int64_t num + ctypedef enum NPY_DATETIMEUNIT: + NPY_FR_Y + NPY_FR_M + NPY_FR_W + NPY_FR_D + NPY_FR_B + NPY_FR_h + NPY_FR_m + NPY_FR_s + NPY_FR_ms + NPY_FR_us + NPY_FR_ns + NPY_FR_ps + NPY_FR_fs + NPY_FR_as + # Typedefs that matches the runtime dtype objects in # the numpy module. From 8cc379da170ad95eb8b5b899ee3e450526576e85 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 18 May 2020 13:23:28 -0700 Subject: [PATCH 0091/4003] move below int64_t declaration --- numpy/__init__.pxd | 78 +++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index bd7f169f5363..fcc4fc8cb3eb 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -689,45 +689,6 @@ cdef extern from "numpy/arrayobject.h": int PyArray_SetBaseObject(ndarray, base) # NOTE: steals a reference to base! Use "set_array_base()" instead. -cdef extern from "numpy/ndarrayobject.h": - PyTypeObject PyTimedeltaArrType_Type - PyTypeObject PyDatetimeArrType_Type - ctypedef int64_t npy_timedelta - ctypedef int64_t npy_datetime - -cdef extern from "numpy/arrayscalars.h": - ctypedef struct PyDatetimeScalarObject: - # PyObject_HEAD - npy_datetime obval - PyArray_DatetimeMetaData obmeta - - ctypedef struct PyTimedeltaScalarObject: - # PyObject_HEAD - npy_timedelta obval - PyArray_DatetimeMetaData obmeta - -cdef extern from "numpy/ndarraytypes.h": - ctypedef struct PyArray_DatetimeMetaData: - NPY_DATETIMEUNIT base - int64_t num - - ctypedef enum NPY_DATETIMEUNIT: - NPY_FR_Y - NPY_FR_M - NPY_FR_W - NPY_FR_D - NPY_FR_B - NPY_FR_h - NPY_FR_m - NPY_FR_s - NPY_FR_ms - NPY_FR_us - NPY_FR_ns - NPY_FR_ps - NPY_FR_fs - NPY_FR_as - - # Typedefs that matches the runtime dtype objects in # the numpy module. @@ -868,6 +829,45 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset return f +cdef extern from "numpy/ndarrayobject.h": + PyTypeObject PyTimedeltaArrType_Type + PyTypeObject PyDatetimeArrType_Type + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime + +cdef extern from "numpy/arrayscalars.h": + ctypedef struct PyDatetimeScalarObject: + # PyObject_HEAD + npy_datetime obval + PyArray_DatetimeMetaData obmeta + + ctypedef struct PyTimedeltaScalarObject: + # PyObject_HEAD + npy_timedelta obval + PyArray_DatetimeMetaData obmeta + +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + + ctypedef enum NPY_DATETIMEUNIT: + NPY_FR_Y + NPY_FR_M + NPY_FR_W + NPY_FR_D + NPY_FR_B + NPY_FR_h + NPY_FR_m + NPY_FR_s + NPY_FR_ms + NPY_FR_us + NPY_FR_ns + NPY_FR_ps + NPY_FR_fs + NPY_FR_as + + # # ufunc API # From 97c157f7dd89f8517f52d865518b4a0a6de5effa Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 18 May 2020 13:42:15 -0700 Subject: [PATCH 0092/4003] troubleshoot CI --- numpy/__init__.pxd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index fcc4fc8cb3eb..c288ba223f12 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -835,6 +835,11 @@ cdef extern from "numpy/ndarrayobject.h": ctypedef int64_t npy_timedelta ctypedef int64_t npy_datetime +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + cdef extern from "numpy/arrayscalars.h": ctypedef struct PyDatetimeScalarObject: # PyObject_HEAD @@ -846,11 +851,6 @@ cdef extern from "numpy/arrayscalars.h": npy_timedelta obval PyArray_DatetimeMetaData obmeta -cdef extern from "numpy/ndarraytypes.h": - ctypedef struct PyArray_DatetimeMetaData: - NPY_DATETIMEUNIT base - int64_t num - ctypedef enum NPY_DATETIMEUNIT: NPY_FR_Y NPY_FR_M From 690a0ab4c65bb828b73526ad9064d7495fdbd6b6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 21 May 2020 15:03:36 -0700 Subject: [PATCH 0093/4003] Fix np.dtype declaration --- numpy/__init__.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index c288ba223f12..c265cdedb96d 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -221,7 +221,7 @@ cdef extern from "numpy/arrayobject.h": cdef int type_num cdef int itemsize "elsize" cdef int alignment - cdef dict fields + cdef object fields cdef tuple names # Use PyDataType_HASSUBARRAY to test whether this field is # valid (the pointer can be NULL). Most users should access From 735f6e0c8a023ae6cd62f792101f3c631994186c Mon Sep 17 00:00:00 2001 From: LSchroefl <65246829+LSchroefl@users.noreply.github.com> Date: Fri, 22 May 2020 00:10:05 +0200 Subject: [PATCH 0094/4003] Update numpy/lib/function_base.py Co-authored-by: Ross Barnowski --- numpy/lib/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index bb6a84795b2f..ee67133dfeb1 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1332,7 +1332,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): If `xp` or `fp` are not 1-D sequences If `period == 0` - See also + See Also -------- scipy.interpolate From 8029c4565a8893b13fa6a5818f1d5888c82fe404 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 21 May 2020 17:10:40 -0500 Subject: [PATCH 0095/4003] BUG: Fix refcounting in add_newdoc this is a trivial refcounting fix, since this function is mostly used only at startup, the small refcounting issue was not noticed until the new tests were added. --- numpy/core/src/multiarray/compiled_base.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index b63bca368661..7a232b5d93aa 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -1444,6 +1444,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) #define _ADDDOC(doc, name) \ if (!(doc)) { \ doc = docstr; \ + Py_INCREF(str); /* hold on to string (leaks reference) */ \ } \ else if (strcmp(doc, docstr) != 0) { \ PyErr_Format(PyExc_RuntimeError, "%s method %s", name, msg); \ @@ -1476,6 +1477,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) doc_attr = PyObject_GetAttrString(obj, "__doc__"); if (doc_attr != NULL && doc_attr != Py_None && (PyUnicode_Compare(doc_attr, str) != 0)) { + Py_DECREF(doc_attr); if (PyErr_Occurred()) { /* error during PyUnicode_Compare */ return NULL; @@ -1495,7 +1497,6 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) #undef _ADDDOC - Py_INCREF(str); Py_RETURN_NONE; } From 0d7909bfb350bf33c86f40636e3b8717e5c41f7e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 21 May 2020 15:54:30 -0700 Subject: [PATCH 0096/4003] First stab at tests, try on the CI --- numpy/__init__.pxd | 6 +- numpy/core/tests/examples/checks.pyx | 26 ++++++ numpy/core/tests/examples/setup.py | 26 ++++++ numpy/core/tests/test_cython.py | 114 +++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 numpy/core/tests/examples/checks.pyx create mode 100644 numpy/core/tests/examples/setup.py create mode 100644 numpy/core/tests/test_cython.py diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index c265cdedb96d..338da3d7e0e7 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -26,7 +26,7 @@ cimport libc.stdio as stdio cdef extern from "Python.h": ctypedef int Py_intptr_t - bint PyObject_TypeCheck(object obj, PyTypeObject* type) nogil + bint PyObject_TypeCheck(object obj, PyTypeObject* type) cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -1018,7 +1018,7 @@ cdef extern from *: """ -cdef inline bint is_timedelta64_object(object obj) nogil: +cdef inline bint is_timedelta64_object(object obj): """ Cython equivalent of `isinstance(obj, np.timedelta64)` @@ -1033,7 +1033,7 @@ cdef inline bint is_timedelta64_object(object obj) nogil: return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) -cdef inline bint is_datetime64_object(object obj) nogil: +cdef inline bint is_datetime64_object(object obj): """ Cython equivalent of `isinstance(obj, np.datetime64)` diff --git a/numpy/core/tests/examples/checks.pyx b/numpy/core/tests/examples/checks.pyx new file mode 100644 index 000000000000..ecf0ad3fa396 --- /dev/null +++ b/numpy/core/tests/examples/checks.pyx @@ -0,0 +1,26 @@ +""" +Functions in this module give python-space wrappers for cython functions +exposed in numpy/__init__.pxd, so they can be tested in test_cython.py +""" +cimport numpy as cnp +cnp.import_array() + + +def is_td64(obj): + return cnp.is_timedelta64_object(obj) + + +def is_dt64(obj): + return cnp.is_datetime64_object(obj) + + +def get_dt64_value(obj): + return cnp.get_datetime64_value(obj) + + +def get_td64_value(obj): + return cnp.get_timedelta64_value(obj) + + +def get_dt64_unit(obj): + return cnp.get_datetime64_unit(obj) diff --git a/numpy/core/tests/examples/setup.py b/numpy/core/tests/examples/setup.py new file mode 100644 index 000000000000..9860bf5f7c75 --- /dev/null +++ b/numpy/core/tests/examples/setup.py @@ -0,0 +1,26 @@ +""" +Provide python-space access to the functions exposed in numpy/__init__.pxd +for testing. +""" + +import numpy as np +from distutils.core import setup +from Cython.Build import cythonize +from setuptools.extension import Extension +import os + +here = os.path.dirname(__file__) +macros = [("NPY_NO_DEPRECATED_API", 0)] + +checks = Extension( + "checks", + sources=[os.path.join(here, "checks.pyx")], + include_dirs=[np.get_include()], + define_macros=macros, +) + +extensions = [checks] + +setup( + ext_modules=cythonize(extensions) +) diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py new file mode 100644 index 000000000000..bb173e4394da --- /dev/null +++ b/numpy/core/tests/test_cython.py @@ -0,0 +1,114 @@ + +import os +import shutil +import subprocess +import sys +import pytest + +import numpy as np + +# This import is copied from random.tests.test_extending +try: + import cython + from Cython.Compiler.Version import version as cython_version +except ImportError: + cython = None +else: + from distutils.version import LooseVersion + # Cython 0.29.14 is required for Python 3.8 and there are + # other fixes in the 0.29 series that are needed even for earlier + # Python versions. + # Note: keep in sync with the one in pyproject.toml + required_version = LooseVersion("0.29.14") + if LooseVersion(cython_version) < required_version: + # too old or wrong cython, skip the test + cython = None + +pytestmark = pytest.mark.skipif(cython is None, reason="requires cython") + + +@pytest.fixture +def install_temp(request, tmp_path): + # Based in part on test_cython from random.tests.test_extending + + here = os.path.dirname(__file__) + ext_dir = os.path.join(here, "examples") + + #assert False + tmp_path = tmp_path._str#str(tmp_path) + cytest = os.path.join(tmp_path, "cytest") + + shutil.copytree(ext_dir, cytest) + # build the examples and "install" them into a temporary directory + + build_dir = os.path.join(tmp_path, "examples") + subprocess.check_call([sys.executable, "setup.py", "build", "install", + "--prefix", os.path.join(tmp_path, "installdir"), + "--single-version-externally-managed", + "--record", os.path.join(tmp_path, "tmp_install_log.txt"), + ], + cwd=cytest, + ) + sys.path.append(cytest) + + +def test_is_timedelta64_object(install_temp): + import checks + + assert checks.is_td64(np.timedelta64(1234)) + assert checks.is_td64(np.timedelta64(1234, "ns")) + assert checks.is_td64(np.timedelta64("NaT", "ns")) + + assert not checks.is_td64(1) + assert not checks.is_td64(None) + assert not checks.is_td64("foo") + assert not checks.is_td64(np.datetime64("now")) + + +def test_is_datetime64_object(install_temp): + import checks + + assert checks.is_dt64(np.datetime64(1234)) + assert checks.is_dt64(np.datetime64(1234, "ns")) + assert checks.is_dt64(np.datetime64("NaT", "ns")) + + assert not checks.is_dt64(1) + assert not checks.is_dt64(None) + assert not checks.is_dt64("foo") + assert not checks.is_dt64(np.timedelta64(1234)) + + +def test_get_datetime64_value(install_temp): + import checks + + dt64 = np.datetime64("2016-01-01", "ns") + + result = checks.get_dt64_value(dt64) + expected = dt64.view("i8") + + assert result == expected + + +def test_get_timedelta64_value(install_temp): + import checks + + td64 = np.timedelta(12345, "h") + + result = checks.get_td64_value(dt64) + expected = td64.view("i8") + + assert result == expected + + +def test_get_datetime64_unit(install_temp): + import checks + + dt64 = np.datetime64("2016-01-01", "ns") + result = checks.get_dt64_unit(dt64) + expected = 11 + assert result == expected + + td64 = np.timedelta(12345, "h") + result = checks.get_dt64_unit(dt64) + expected = 5 + assert result == expected From 1a3aa8416033e052e4353548d18a447755afaab4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 21 May 2020 18:41:24 -0500 Subject: [PATCH 0097/4003] CI: Create a link for the circleCI artifact --- .github/workflows/circleci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/circleci.yml diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml new file mode 100644 index 000000000000..de02ac6d330d --- /dev/null +++ b/.github/workflows/circleci.yml @@ -0,0 +1,12 @@ +on: [status] +jobs: + circleci_artifacts_redirector_job: + runs-on: ubuntu-latest + name: Run CircleCI artifacts redirector + steps: + - name: GitHub Action step + uses: larsoner/circleci-artifacts-redirector-action@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + artifact-path: 0/doc/build/html/index.html + circleci-jobs: build From 586bc6002b66624ff17db4213381e985cea80cf2 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 22 May 2020 09:23:59 +0300 Subject: [PATCH 0098/4003] ENH: run tests for __init__.pxd --- numpy/core/setup.py | 1 + numpy/core/tests/test_cython.py | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index fcc4225459d6..2b850c167f68 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -963,6 +963,7 @@ def generate_umath_c(ext, build_dir): config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_dir('tests/examples') config.make_svn_version_py() diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index bb173e4394da..e7d3c0f0129b 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -35,21 +35,31 @@ def install_temp(request, tmp_path): ext_dir = os.path.join(here, "examples") #assert False - tmp_path = tmp_path._str#str(tmp_path) + tmp_path = tmp_path._str # str(tmp_path) cytest = os.path.join(tmp_path, "cytest") shutil.copytree(ext_dir, cytest) # build the examples and "install" them into a temporary directory build_dir = os.path.join(tmp_path, "examples") + install_log = os.path.join(tmp_path, "tmp_install_log.txt") subprocess.check_call([sys.executable, "setup.py", "build", "install", "--prefix", os.path.join(tmp_path, "installdir"), "--single-version-externally-managed", - "--record", os.path.join(tmp_path, "tmp_install_log.txt"), + "--record", install_log, ], cwd=cytest, ) - sys.path.append(cytest) + + # In order to import the built module, we need its path to sys.path + # so parse that out of the record + with open(install_log) as fid: + for line in fid: + if 'checks' in line: + sys.path.append(os.path.dirname(line)) + break + else: + raise RuntimeError(f'could not parse "{install_log}"') def test_is_timedelta64_object(install_temp): @@ -92,9 +102,9 @@ def test_get_datetime64_value(install_temp): def test_get_timedelta64_value(install_temp): import checks - td64 = np.timedelta(12345, "h") + td64 = np.timedelta64(12345, "h") - result = checks.get_td64_value(dt64) + result = checks.get_td64_value(td64) expected = td64.view("i8") assert result == expected @@ -108,7 +118,7 @@ def test_get_datetime64_unit(install_temp): expected = 11 assert result == expected - td64 = np.timedelta(12345, "h") + td64 = np.timedelta64(12345, "h") result = checks.get_dt64_unit(dt64) expected = 5 assert result == expected From 1e7ef595cf6ec20c321a93387916f05309f3b4fa Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 22 May 2020 10:05:43 -0500 Subject: [PATCH 0099/4003] BUG: Fix dtype leak in `PyArray_FromAny` error path Also adds a test to bincount which will run into this path. The leak can be triggered by using a reference count checker on the test suit (e.g. pytest-leaks). Closes gh-16339 --- numpy/core/src/multiarray/ctors.c | 2 ++ numpy/lib/tests/test_function_base.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 502ab0ea916e..9e8646b25d81 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -2029,12 +2029,14 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, PyErr_SetString(PyExc_ValueError, "object of too small depth for desired array"); Py_DECREF(arr); + Py_XDECREF(newtype); ret = NULL; } else if (max_depth != 0 && PyArray_NDIM(arr) > max_depth) { PyErr_SetString(PyExc_ValueError, "object too deep for desired array"); Py_DECREF(arr); + Py_XDECREF(newtype); ret = NULL; } else { diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 008ea0759db9..9ba0be56ab0a 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -2423,6 +2423,15 @@ def test_dtype_reference_leaks(self): assert_equal(sys.getrefcount(np.dtype(np.intp)), intp_refcount) assert_equal(sys.getrefcount(np.dtype(np.double)), double_refcount) + @pytest.mark.parametrize("vals", [[[2, 2]], 2]) + def test_error_not_1d(self, vals): + # Test that values has to be 1-D (both as array and nested list) + vals_arr = np.asarray(vals) + with assert_raises(ValueError): + np.bincount(vals_arr) + with assert_raises(ValueError): + np.bincount(vals) + class TestInterp: From dabf31c74f6f3153ef4e7c72ad969c37f8652c8a Mon Sep 17 00:00:00 2001 From: Wojciech Rzadkowski <33913808+wrzadkow@users.noreply.github.com> Date: Fri, 22 May 2020 17:43:08 +0200 Subject: [PATCH 0100/4003] MAINT: Remove f-strings in setup.py. (gh-16346) Remove f-strings from setup.py to allow for an informative error message for python<3.6 users. Closes #16345. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a86e28341145..8178a6c51cb1 100755 --- a/setup.py +++ b/setup.py @@ -179,7 +179,7 @@ def check_submodules(): if 'path' in l: p = l.split('=')[-1].strip() if not os.path.exists(p): - raise ValueError(f'Submodule {p} missing') + raise ValueError('Submodule {} missing'.format(p)) proc = subprocess.Popen(['git', 'submodule', 'status'], @@ -188,7 +188,7 @@ def check_submodules(): status = status.decode("ascii", "replace") for line in status.splitlines(): if line.startswith('-') or line.startswith('+'): - raise ValueError(f'Submodule not clean: {line}') + raise ValueError('Submodule not clean: {}'.format(line)) From 78593a1059f0a9c04385f97b2c1caa221efefa5f Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Fri, 22 May 2020 19:26:27 +0300 Subject: [PATCH 0101/4003] BUILD: Remove Accelerate support (#15759) Remove support for Apple Accelerate, since it is buggy. A build error should occur on most or all setups if linked against Accelerate. Test or import failures should occur on setups where Accelerate is picked up dynamically. Co-authored-by: Warren Weckesser --- .travis.yml | 4 +- MANIFEST.in | 1 - azure-pipelines.yml | 12 +- .../upcoming_changes/15759.improvement.rst | 4 + doc/source/user/building.rst | 6 +- numpy/_build_utils/README | 8 - numpy/_build_utils/__init__.py | 0 numpy/_build_utils/apple_accelerate.py | 26 -- numpy/_build_utils/src/apple_sgemv_fix.c | 253 ------------------ numpy/core/setup.py | 13 +- numpy/core/tests/test_multiarray.py | 64 ----- numpy/distutils/system_info.py | 26 +- numpy/linalg/setup.py | 8 +- 13 files changed, 51 insertions(+), 374 deletions(-) create mode 100644 doc/release/upcoming_changes/15759.improvement.rst delete mode 100644 numpy/_build_utils/README delete mode 100644 numpy/_build_utils/__init__.py delete mode 100644 numpy/_build_utils/apple_accelerate.py delete mode 100644 numpy/_build_utils/src/apple_sgemv_fix.c diff --git a/.travis.yml b/.travis.yml index e019495fb852..aa01457fbd80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,8 +71,8 @@ jobs: - BLAS=None - LAPACK=None - ATLAS=None - - NPY_BLAS_ORDER=mkl,blis,openblas,atlas,accelerate,blas - - NPY_LAPACK_ORDER=MKL,OPENBLAS,ATLAS,ACCELERATE,LAPACK + - NPY_BLAS_ORDER=mkl,blis,openblas,atlas,blas + - NPY_LAPACK_ORDER=MKL,OPENBLAS,ATLAS,LAPACK - USE_ASV=1 - python: 3.7 diff --git a/MANIFEST.in b/MANIFEST.in index b58f85d4d295..f710c92e6a19 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -21,7 +21,6 @@ include numpy/__init__.pxd # Note that sub-directories that don't have __init__ are apparently not # included by 'recursive-include', so list those separately recursive-include numpy * -recursive-include numpy/_build_utils * recursive-include numpy/linalg/lapack_lite * recursive-include tools * # Add sdist files whose use depends on local configuration. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 564f5d8e89a8..ea2b414b0348 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -75,10 +75,13 @@ stages: PYTHON_VERSION: '3.6' NPY_USE_BLAS_ILP64: '1' USE_OPENBLAS: '1' - USE_XCODE_10: '1' - Accelerate: - PYTHON_VERSION: '3.6' - USE_OPENBLAS: '0' + # Disable this job: the azure images do not create the problematic + # symlink from Accelerate to OpenBLAS. We still have the test + # at import to detect a buggy Accelerate, just cannot easily trigger + # it with azure. + # Accelerate: + # PYTHON_VERSION: '3.6' + # USE_OPENBLAS: '0' steps: # the @0 refers to the (major) version of the *task* on Microsoft's @@ -145,7 +148,6 @@ stages: BLAS: None LAPACK: None ATLAS: None - ACCELERATE: None CC: /usr/bin/clang condition: eq(variables['USE_OPENBLAS'], '1') - script: python setup.py build -j 4 build_ext --inplace install diff --git a/doc/release/upcoming_changes/15759.improvement.rst b/doc/release/upcoming_changes/15759.improvement.rst new file mode 100644 index 000000000000..0a1b255f7e7c --- /dev/null +++ b/doc/release/upcoming_changes/15759.improvement.rst @@ -0,0 +1,4 @@ +Remove the Accelerate library as a candidate LAPACK library +----------------------------------------------------------- +Apple no longer supports Accelerate. Remove it. + diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index 546fd7dafa43..47a0a03c9958 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -123,8 +123,7 @@ The default order for the libraries are: 2. BLIS 3. OpenBLAS 4. ATLAS -5. Accelerate (MacOS) -6. BLAS (NetLIB) +5. BLAS (NetLIB) If you wish to build against OpenBLAS but you also have BLIS available one may predefine the order of searching via the environment variable @@ -146,8 +145,7 @@ The default order for the libraries are: 2. OpenBLAS 3. libFLAME 4. ATLAS -5. Accelerate (MacOS) -6. LAPACK (NetLIB) +5. LAPACK (NetLIB) If you wish to build against OpenBLAS but you also have MKL available one diff --git a/numpy/_build_utils/README b/numpy/_build_utils/README deleted file mode 100644 index 6976e0233996..000000000000 --- a/numpy/_build_utils/README +++ /dev/null @@ -1,8 +0,0 @@ -======= -WARNING -======= - -This directory (numpy/_build_utils) is *not* part of the public numpy API, - - it is internal build support for numpy. - - it is only present in source distributions or during an in place build - - it is *not* installed with the rest of numpy diff --git a/numpy/_build_utils/__init__.py b/numpy/_build_utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/numpy/_build_utils/apple_accelerate.py b/numpy/_build_utils/apple_accelerate.py deleted file mode 100644 index b26aa12ad978..000000000000 --- a/numpy/_build_utils/apple_accelerate.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -import re - -__all__ = ['uses_accelerate_framework', 'get_sgemv_fix'] - -def uses_accelerate_framework(info): - """ Returns True if Accelerate framework is used for BLAS/LAPACK """ - # If we're not building on Darwin (macOS), don't use Accelerate - if sys.platform != "darwin": - return False - # If we're building on macOS, but targeting a different platform, - # don't use Accelerate. - if os.getenv('_PYTHON_HOST_PLATFORM', None): - return False - r_accelerate = re.compile("Accelerate") - extra_link_args = info.get('extra_link_args', '') - for arg in extra_link_args: - if r_accelerate.search(arg): - return True - return False - -def get_sgemv_fix(): - """ Returns source file needed to correct SGEMV """ - path = os.path.abspath(os.path.dirname(__file__)) - return [os.path.join(path, 'src', 'apple_sgemv_fix.c')] diff --git a/numpy/_build_utils/src/apple_sgemv_fix.c b/numpy/_build_utils/src/apple_sgemv_fix.c deleted file mode 100644 index b1dbeb681845..000000000000 --- a/numpy/_build_utils/src/apple_sgemv_fix.c +++ /dev/null @@ -1,253 +0,0 @@ -/* This is a collection of ugly hacks to circumvent a bug in - * Apple Accelerate framework's SGEMV subroutine. - * - * See: https://github.com/numpy/numpy/issues/4007 - * - * SGEMV in Accelerate framework will segfault on MacOS X version 10.9 - * (aka Mavericks) if arrays are not aligned to 32 byte boundaries - * and the CPU supports AVX instructions. This can produce segfaults - * in np.dot. - * - * This patch overshadows the symbols cblas_sgemv, sgemv_ and sgemv - * exported by Accelerate to produce the correct behavior. The MacOS X - * version and CPU specs are checked on module import. If Mavericks and - * AVX are detected the call to SGEMV is emulated with a call to SGEMM - * if the arrays are not 32 byte aligned. If the exported symbols cannot - * be overshadowed on module import, a fatal error is produced and the - * process aborts. All the fixes are in a self-contained C file - * and do not alter the multiarray C code. The patch is not applied - * unless NumPy is configured to link with Apple's Accelerate - * framework. - * - */ - -#define NPY_NO_DEPRECATED_API NPY_API_VERSION -#include "Python.h" -#include "numpy/arrayobject.h" - -#include -#include -#include -#include -#include -#include -#include - -/* ----------------------------------------------------------------- */ -/* Original cblas_sgemv */ - -#define VECLIB_FILE "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/vecLib" - -enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102}; -enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113}; -extern void cblas_xerbla(int info, const char *rout, const char *form, ...); - -typedef void cblas_sgemv_t(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const float alpha, const float *A, const int lda, - const float *X, const int incX, - const float beta, float *Y, const int incY); - -typedef void cblas_sgemm_t(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_TRANSPOSE TransB, - const int M, const int N, const int K, - const float alpha, const float *A, const int lda, - const float *B, const int ldb, - const float beta, float *C, const int incC); - -typedef void fortran_sgemv_t( const char* trans, const int* m, const int* n, - const float* alpha, const float* A, const int* ldA, - const float* X, const int* incX, - const float* beta, float* Y, const int* incY ); - -static void *veclib = NULL; -static cblas_sgemv_t *accelerate_cblas_sgemv = NULL; -static cblas_sgemm_t *accelerate_cblas_sgemm = NULL; -static fortran_sgemv_t *accelerate_sgemv = NULL; -static int AVX_and_10_9 = 0; - -/* Dynamic check for AVX support - * __builtin_cpu_supports("avx") is available in gcc 4.8, - * but clang and icc do not currently support it. */ -static inline int -cpu_supports_avx() -{ - int enabled, r; - size_t length = sizeof(enabled); - r = sysctlbyname("hw.optional.avx1_0", &enabled, &length, NULL, 0); - if ( r == 0 && enabled != 0) { - return 1; - } - else { - return 0; - } -} - -/* Check if we are using MacOS X version 10.9 */ -static inline int -using_mavericks() -{ - int r; - char str[32] = {0}; - size_t size = sizeof(str); - r = sysctlbyname("kern.osproductversion", str, &size, NULL, 0); - if ( r == 0 && strncmp(str, "10.9", strlen("10.9")) == 0) { - return 1; - } - else { - return 0; - } -} - -__attribute__((destructor)) -static void unloadlib(void) -{ - if (veclib) dlclose(veclib); -} - -__attribute__((constructor)) -static void loadlib() -/* automatically executed on module import */ -{ - char errormsg[1024]; - int AVX, MAVERICKS; - memset((void*)errormsg, 0, sizeof(errormsg)); - /* check if the CPU supports AVX */ - AVX = cpu_supports_avx(); - /* check if the OS is MacOS X Mavericks */ - MAVERICKS = using_mavericks(); - /* we need the workaround when the CPU supports - * AVX and the OS version is Mavericks */ - AVX_and_10_9 = AVX && MAVERICKS; - /* load vecLib */ - veclib = dlopen(VECLIB_FILE, RTLD_LOCAL | RTLD_FIRST); - if (!veclib) { - veclib = NULL; - snprintf(errormsg, sizeof(errormsg), - "Failed to open vecLib from location '%s'.", VECLIB_FILE); - Py_FatalError(errormsg); /* calls abort() and dumps core */ - } - /* resolve Fortran SGEMV from Accelerate */ - accelerate_sgemv = (fortran_sgemv_t*) dlsym(veclib, "sgemv_"); - if (!accelerate_sgemv) { - unloadlib(); - Py_FatalError("Failed to resolve symbol 'sgemv_'."); - } - /* resolve cblas_sgemv from Accelerate */ - accelerate_cblas_sgemv = (cblas_sgemv_t*) dlsym(veclib, "cblas_sgemv"); - if (!accelerate_cblas_sgemv) { - unloadlib(); - Py_FatalError("Failed to resolve symbol 'cblas_sgemv'."); - } - /* resolve cblas_sgemm from Accelerate */ - accelerate_cblas_sgemm = (cblas_sgemm_t*) dlsym(veclib, "cblas_sgemm"); - if (!accelerate_cblas_sgemm) { - unloadlib(); - Py_FatalError("Failed to resolve symbol 'cblas_sgemm'."); - } -} - -/* ----------------------------------------------------------------- */ -/* Fortran SGEMV override */ - -void sgemv_( const char* trans, const int* m, const int* n, - const float* alpha, const float* A, const int* ldA, - const float* X, const int* incX, - const float* beta, float* Y, const int* incY ) -{ - /* It is safe to use the original SGEMV if we are not using AVX on Mavericks - * or the input arrays A, X and Y are all aligned on 32 byte boundaries. */ - #define BADARRAY(x) (((npy_intp)(void*)x) % 32) - const int use_sgemm = AVX_and_10_9 && (BADARRAY(A) || BADARRAY(X) || BADARRAY(Y)); - if (!use_sgemm) { - accelerate_sgemv(trans,m,n,alpha,A,ldA,X,incX,beta,Y,incY); - return; - } - - /* Arrays are misaligned, the CPU supports AVX, and we are running - * Mavericks. - * - * Emulation of SGEMV with SGEMM: - * - * SGEMV allows vectors to be strided. SGEMM requires all arrays to be - * contiguous along the leading dimension. To emulate striding in SGEMV - * with the leading dimension arguments in SGEMM we compute - * - * Y = alpha * op(A) @ X + beta * Y - * - * as - * - * Y.T = alpha * X.T @ op(A).T + beta * Y.T - * - * Because Fortran uses column major order and X.T and Y.T are row vectors, - * the leading dimensions of X.T and Y.T in SGEMM become equal to the - * strides of the column vectors X and Y in SGEMV. */ - - switch (*trans) { - case 'T': - case 't': - case 'C': - case 'c': - accelerate_cblas_sgemm( CblasColMajor, CblasNoTrans, CblasNoTrans, - 1, *n, *m, *alpha, X, *incX, A, *ldA, *beta, Y, *incY ); - break; - case 'N': - case 'n': - accelerate_cblas_sgemm( CblasColMajor, CblasNoTrans, CblasTrans, - 1, *m, *n, *alpha, X, *incX, A, *ldA, *beta, Y, *incY ); - break; - default: - cblas_xerbla(1, "SGEMV", "Illegal transpose setting: %c\n", *trans); - } -} - -/* ----------------------------------------------------------------- */ -/* Override for an alias symbol for sgemv_ in Accelerate */ - -void sgemv (char *trans, - const int *m, const int *n, - const float *alpha, - const float *A, const int *lda, - const float *B, const int *incB, - const float *beta, - float *C, const int *incC) -{ - sgemv_(trans,m,n,alpha,A,lda,B,incB,beta,C,incC); -} - -/* ----------------------------------------------------------------- */ -/* cblas_sgemv override, based on Netlib CBLAS code */ - -void cblas_sgemv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const float alpha, const float *A, const int lda, - const float *X, const int incX, const float beta, - float *Y, const int incY) -{ - char TA; - if (order == CblasColMajor) - { - if (TransA == CblasNoTrans) TA = 'N'; - else if (TransA == CblasTrans) TA = 'T'; - else if (TransA == CblasConjTrans) TA = 'C'; - else - { - cblas_xerbla(2, "cblas_sgemv","Illegal TransA setting, %d\n", TransA); - } - sgemv_(&TA, &M, &N, &alpha, A, &lda, X, &incX, &beta, Y, &incY); - } - else if (order == CblasRowMajor) - { - if (TransA == CblasNoTrans) TA = 'T'; - else if (TransA == CblasTrans) TA = 'N'; - else if (TransA == CblasConjTrans) TA = 'N'; - else - { - cblas_xerbla(2, "cblas_sgemv", "Illegal TransA setting, %d\n", TransA); - return; - } - sgemv_(&TA, &N, &M, &alpha, A, &lda, X, &incX, &beta, Y, &incY); - } - else - cblas_xerbla(1, "cblas_sgemv", "Illegal Order setting, %d\n", order); -} diff --git a/numpy/core/setup.py b/numpy/core/setup.py index fcc4225459d6..bf807641d2e8 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -10,9 +10,6 @@ from numpy.distutils import log from distutils.dep_util import newer from distutils.sysconfig import get_config_var -from numpy._build_utils.apple_accelerate import ( - uses_accelerate_framework, get_sgemv_fix - ) from numpy.compat import npy_load_module from setup_common import * # noqa: F403 @@ -392,7 +389,13 @@ def visibility_define(config): def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration, dot_join - from numpy.distutils.system_info import get_info + from numpy.distutils.system_info import (get_info, blas_opt_info, + lapack_opt_info) + + # Accelerate is buggy, disallow it. See also numpy/linalg/setup.py + for opt_order in (blas_opt_info.blas_order, lapack_opt_info.lapack_order): + if 'accelerate' in opt_order: + opt_order.remove('accelerate') config = Configuration('core', parent_package, top_path) local_dir = config.local_path @@ -762,8 +765,6 @@ def get_mathlib_info(*args): common_src.extend([join('src', 'common', 'cblasfuncs.c'), join('src', 'common', 'python_xerbla.c'), ]) - if uses_accelerate_framework(blas_info): - common_src.extend(get_sgemv_fix()) else: extra_info = {} diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 2edcc680bdfd..ab5ec266eff1 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -5882,70 +5882,6 @@ def test_dot_array_order(self): assert_equal(np.dot(b, a), res) assert_equal(np.dot(b, b), res) - def test_accelerate_framework_sgemv_fix(self): - - def aligned_array(shape, align, dtype, order='C'): - d = dtype(0) - N = np.prod(shape) - tmp = np.zeros(N * d.nbytes + align, dtype=np.uint8) - address = tmp.__array_interface__["data"][0] - for offset in range(align): - if (address + offset) % align == 0: - break - tmp = tmp[offset:offset+N*d.nbytes].view(dtype=dtype) - return tmp.reshape(shape, order=order) - - def as_aligned(arr, align, dtype, order='C'): - aligned = aligned_array(arr.shape, align, dtype, order) - aligned[:] = arr[:] - return aligned - - def assert_dot_close(A, X, desired): - assert_allclose(np.dot(A, X), desired, rtol=1e-5, atol=1e-7) - - m = aligned_array(100, 15, np.float32) - s = aligned_array((100, 100), 15, np.float32) - np.dot(s, m) # this will always segfault if the bug is present - - testdata = itertools.product((15,32), (10000,), (200,89), ('C','F')) - for align, m, n, a_order in testdata: - # Calculation in double precision - A_d = np.random.rand(m, n) - X_d = np.random.rand(n) - desired = np.dot(A_d, X_d) - # Calculation with aligned single precision - A_f = as_aligned(A_d, align, np.float32, order=a_order) - X_f = as_aligned(X_d, align, np.float32) - assert_dot_close(A_f, X_f, desired) - # Strided A rows - A_d_2 = A_d[::2] - desired = np.dot(A_d_2, X_d) - A_f_2 = A_f[::2] - assert_dot_close(A_f_2, X_f, desired) - # Strided A columns, strided X vector - A_d_22 = A_d_2[:, ::2] - X_d_2 = X_d[::2] - desired = np.dot(A_d_22, X_d_2) - A_f_22 = A_f_2[:, ::2] - X_f_2 = X_f[::2] - assert_dot_close(A_f_22, X_f_2, desired) - # Check the strides are as expected - if a_order == 'F': - assert_equal(A_f_22.strides, (8, 8 * m)) - else: - assert_equal(A_f_22.strides, (8 * n, 8)) - assert_equal(X_f_2.strides, (8,)) - # Strides in A rows + cols only - X_f_2c = as_aligned(X_f_2, align, np.float32) - assert_dot_close(A_f_22, X_f_2c, desired) - # Strides just in A cols - A_d_12 = A_d[:, ::2] - desired = np.dot(A_d_12, X_d_2) - A_f_12 = A_f[:, ::2] - assert_dot_close(A_f_12, X_f_2c, desired) - # Strides in A cols and X - assert_dot_close(A_f_12, X_f_2, desired) - class MatmulCommon: """Common tests for '@' operator and numpy.matmul. diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 3a6a7b29da97..df82683dc18f 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -362,6 +362,22 @@ def add_system_root(library_root): so_ext = get_shared_lib_extension() +def is_symlink_to_accelerate(filename): + accelpath = '/System/Library/Frameworks/Accelerate.framework' + return (sys.platform == 'darwin' and os.path.islink(filename) and + os.path.realpath(filename).startswith(accelpath)) + + +_accel_msg = ( + 'Found {filename}, but that file is a symbolic link to the ' + 'MacOS Accelerate framework, which is not supported by NumPy. ' + 'You must configure the build to use a different optimized library, ' + 'or disable the use of optimized BLAS and LAPACK by setting the ' + 'environment variables NPY_BLAS_ORDER="" and NPY_LAPACK_ORDER="" ' + 'before building NumPy.' +) + + def get_standard_file(fname): """Returns a list of files named 'fname' from 1) System-wide directory (directory-location of this module) @@ -427,7 +443,6 @@ def get_info(name, notfound_action=0): 'blis': blis_info, # use blas_opt instead 'lapack_mkl': lapack_mkl_info, # use lapack_opt instead 'blas_mkl': blas_mkl_info, # use blas_opt instead - 'accelerate': accelerate_info, # use blas_opt instead 'openblas64_': openblas64__info, 'openblas64__lapack': openblas64__lapack_info, 'openblas_ilp64': openblas_ilp64_info, @@ -919,6 +934,9 @@ def _find_lib(self, lib_dir, lib, exts): for prefix in lib_prefixes: p = self.combine_paths(lib_dir, prefix + lib + ext) if p: + # p[0] is the full path to the binary library file. + if is_symlink_to_accelerate(p[0]): + raise RuntimeError(_accel_msg.format(filename=p[0])) break if p: assert len(p) == 1 @@ -1650,8 +1668,8 @@ def get_atlas_version(**config): class lapack_opt_info(system_info): notfounderror = LapackNotFoundError - # List of all known BLAS libraries, in the default order - lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'accelerate', 'lapack'] + # List of all known LAPACK libraries, in the default order + lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'lapack'] order_env_var_name = 'NPY_LAPACK_ORDER' def _calc_info_mkl(self): @@ -1823,7 +1841,7 @@ class lapack64__opt_info(lapack_ilp64_opt_info): class blas_opt_info(system_info): notfounderror = BlasNotFoundError # List of all known BLAS libraries, in the default order - blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'accelerate', 'blas'] + blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'blas'] order_env_var_name = 'NPY_BLAS_ORDER' def _calc_info_mkl(self): diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 57fdd502b485..bb070ed9d892 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -3,11 +3,17 @@ def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration - from numpy.distutils.system_info import get_info, system_info + from numpy.distutils.system_info import ( + get_info, system_info, lapack_opt_info, blas_opt_info) config = Configuration('linalg', parent_package, top_path) config.add_subpackage('tests') + # Accelerate is buggy, disallow it. See also numpy/core/setup.py + for opt_order in (blas_opt_info.blas_order, lapack_opt_info.lapack_order): + if 'accelerate' in opt_order: + opt_order.remove('accelerate') + # Configure lapack_lite src_dir = 'lapack_lite' From ccc2427f8f3d5d7605c822b6143cfbc8058c0f3c Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 22 May 2020 12:36:33 -0400 Subject: [PATCH 0102/4003] BUG: Indentation for docstrings --- numpy/lib/tests/test_utils.py | 20 +++++++++++++++----- numpy/lib/utils.py | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index c96bf795a22e..261cfef5d680 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -49,7 +49,7 @@ def old_func5(self, x): Bizarre indentation. """ return x -new_func5 = deprecate(old_func5) +new_func5 = deprecate(old_func5, message="This function is\ndeprecated.") def old_func6(self, x): @@ -74,10 +74,20 @@ def test_deprecate_fn(): @pytest.mark.skipif(sys.flags.optimize == 2, reason="-OO discards docstrings") -def test_deprecate_help_indentation(): - _compare_docs(old_func4, new_func4) - _compare_docs(old_func5, new_func5) - _compare_docs(old_func6, new_func6) +@pytest.mark.parametrize('old_func, new_func', [ + (old_func4, new_func4), + (old_func5, new_func5), + (old_func6, new_func6), +]) +def test_deprecate_help_indentation(old_func, new_func): + _compare_docs(old_func, new_func) + # Ensure we don't mess up the indentation + for knd, func in (('old', old_func), ('new', new_func)): + for li, line in enumerate(func.__doc__.split('\n')): + if li == 0: + assert line.startswith(' ') or not line.startswith(' '), knd + elif line: + assert line.startswith(' '), knd def _compare_docs(old_func, new_func): diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index f233c7240967..ba8616aa3c20 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -1,5 +1,6 @@ import os import sys +import textwrap import types import re import warnings @@ -117,6 +118,7 @@ def newfunc(*args,**kwds): break skip += len(line) + 1 doc = doc[skip:] + depdoc = textwrap.indent(depdoc, ' ' * indent) doc = '\n\n'.join([depdoc, doc]) newfunc.__doc__ = doc try: From a9652077be95f83f56c9b77e7ad1ed7710516626 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 22 May 2020 13:32:51 -0500 Subject: [PATCH 0103/4003] BUG: Fix reference count leak in array allocation memory path --- numpy/core/src/multiarray/ctors.c | 10 +++++----- numpy/core/tests/test_multiarray.py | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 502ab0ea916e..beee985f60c1 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -869,7 +869,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } -static PyObject * +static void raise_memory_error(int nd, npy_intp const *dims, PyArray_Descr *descr) { static PyObject *exc_type = NULL; @@ -894,12 +894,12 @@ raise_memory_error(int nd, npy_intp const *dims, PyArray_Descr *descr) } PyErr_SetObject(exc_type, exc_value); Py_DECREF(exc_value); - return NULL; + return; fail: /* we couldn't raise the formatted exception for some reason */ PyErr_WriteUnraisable(NULL); - return PyErr_NoMemory(); + PyErr_NoMemory(); } /* @@ -1079,10 +1079,10 @@ PyArray_NewFromDescr_int( data = npy_alloc_cache(nbytes); } if (data == NULL) { - return raise_memory_error(fa->nd, fa->dimensions, descr); + raise_memory_error(fa->nd, fa->dimensions, descr); + goto fail; } fa->flags |= NPY_ARRAY_OWNDATA; - } else { /* diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 2edcc680bdfd..86a8cff0cd07 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -845,6 +845,13 @@ def test_too_big_error(self): assert_raises(ValueError, np.zeros, shape, dtype=np.int8) assert_raises(ValueError, np.ones, shape, dtype=np.int8) + @pytest.mark.skipif(np.dtype(np.intp).itemsize != 8, + reason="malloc may not fail on 32 bit systems") + def test_malloc_fails(self): + # This test is guaranteed to fail due to a too large allocation + with assert_raises(np.core._exceptions._ArrayMemoryError): + np.empty(np.iinfo(np.intp).max, dtype=np.uint8) + def test_zeros(self): types = np.typecodes['AllInteger'] + np.typecodes['AllFloat'] for dt in types: From ce3d79a7c41e6be0a3ad0e8e3214f96fe29c77f9 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 22 May 2020 13:33:13 -0500 Subject: [PATCH 0104/4003] BUG: Add missing dimensions free in `empty_like` with shape --- numpy/core/src/multiarray/multiarraymodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 6915371d8860..a9f673d93a23 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1872,6 +1872,7 @@ array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) /* steals the reference to dtype if it's not NULL */ ret = (PyArrayObject *)PyArray_NewLikeArrayWithShape(prototype, order, dtype, shape.len, shape.ptr, subok); + npy_free_cache_dim_obj(shape); if (!ret) { goto fail; } From 8e3cfd83fb778b03c8bd1537a6d9437461c48584 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 22 May 2020 14:17:43 -0700 Subject: [PATCH 0105/4003] blackify, troubleshoot tests --- numpy/core/tests/test_cython.py | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index e7d3c0f0129b..92ef09c9b1f3 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -1,4 +1,3 @@ - import os import shutil import subprocess @@ -15,6 +14,7 @@ cython = None else: from distutils.version import LooseVersion + # Cython 0.29.14 is required for Python 3.8 and there are # other fixes in the 0.29 series that are needed even for earlier # Python versions. @@ -34,28 +34,33 @@ def install_temp(request, tmp_path): here = os.path.dirname(__file__) ext_dir = os.path.join(here, "examples") - #assert False - tmp_path = tmp_path._str # str(tmp_path) + tmp_path = tmp_path._str cytest = os.path.join(tmp_path, "cytest") shutil.copytree(ext_dir, cytest) # build the examples and "install" them into a temporary directory - - build_dir = os.path.join(tmp_path, "examples") + install_log = os.path.join(tmp_path, "tmp_install_log.txt") - subprocess.check_call([sys.executable, "setup.py", "build", "install", - "--prefix", os.path.join(tmp_path, "installdir"), - "--single-version-externally-managed", - "--record", install_log, - ], - cwd=cytest, - ) + subprocess.check_call( + [ + sys.executable, + "setup.py", + "build", + "install", + "--prefix", + os.path.join(tmp_path, "installdir"), + "--single-version-externally-managed", + "--record", + install_log, + ], + cwd=cytest, + ) # In order to import the built module, we need its path to sys.path # so parse that out of the record with open(install_log) as fid: for line in fid: - if 'checks' in line: + if "checks" in line: sys.path.append(os.path.dirname(line)) break else: @@ -72,13 +77,12 @@ def test_is_timedelta64_object(install_temp): assert not checks.is_td64(1) assert not checks.is_td64(None) assert not checks.is_td64("foo") - assert not checks.is_td64(np.datetime64("now")) + assert not checks.is_td64(np.datetime64("now", "s")) def test_is_datetime64_object(install_temp): import checks - assert checks.is_dt64(np.datetime64(1234)) assert checks.is_dt64(np.datetime64(1234, "ns")) assert checks.is_dt64(np.datetime64("NaT", "ns")) @@ -115,10 +119,10 @@ def test_get_datetime64_unit(install_temp): dt64 = np.datetime64("2016-01-01", "ns") result = checks.get_dt64_unit(dt64) - expected = 11 + expected = 10 assert result == expected td64 = np.timedelta64(12345, "h") - result = checks.get_dt64_unit(dt64) + result = checks.get_dt64_unit(td64) expected = 5 assert result == expected From 471a2ceaf3f0b77ca63003b196e8ba8c54283806 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Fri, 22 May 2020 17:19:04 -0700 Subject: [PATCH 0106/4003] MAINT: Cleanup 'tools/download-wheels.py' (#16329) * ENH: Create download dir if not present for download-wheels * TST: Add tests for download-wheels * MAINT: use exist_ok=True for os.makedirs in download-wheels Co-authored-by: Ross Barnowski * MAINT: Remove test for download_wheels * MAINT: Remove directory creation and error if dir not present * MAINT: Print total files download and remove err for 0 search results Co-authored-by: Ross Barnowski --- tools/download-wheels.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/download-wheels.py b/tools/download-wheels.py index 276fcc6b268f..1e26e0e630e2 100644 --- a/tools/download-wheels.py +++ b/tools/download-wheels.py @@ -31,7 +31,7 @@ def get_wheel_names(version): """ http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED') - tmpl = re.compile(PREFIX + version + '.*\.whl$') + tmpl = re.compile(rf"{PREFIX}{version}.*\.whl$") index_url = f"{STAGING_URL}/files" index_html = http.request('GET', index_url) soup = BeautifulSoup(index_html.data, 'html.parser') @@ -54,13 +54,15 @@ def download_wheels(version, wheelhouse): """ http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED') wheel_names = get_wheel_names(version) - for wheel_name in wheel_names: + + for i, wheel_name in enumerate(wheel_names): wheel_url = f"{STAGING_URL}/{version}/download/{wheel_name}" wheel_path = os.path.join(wheelhouse, wheel_name) with open(wheel_path, 'wb') as f: with http.request('GET', wheel_url, preload_content=False,) as r: - print(f"Downloading {wheel_name}") + print(f"Downloading wheel {i + 1}, name: {wheel_name}") shutil.copyfileobj(r, f) + print(f"\nTotal files downloaded: {len(wheel_names)}") if __name__ == '__main__': @@ -77,4 +79,9 @@ def download_wheels(version, wheelhouse): args = parser.parse_args() wheelhouse = os.path.expanduser(args.wheelhouse) + if not os.path.isdir(wheelhouse): + raise RuntimeError( + f"{wheelhouse} wheelhouse directory is not present." + " Perhaps you need to use the '-w' flag to specify one.") + download_wheels(args.version, wheelhouse) From daffccfa95251024a814139a622feb367e7c74c0 Mon Sep 17 00:00:00 2001 From: Chunlin Date: Sun, 24 May 2020 04:15:17 +0800 Subject: [PATCH 0107/4003] DOC: Reconstruct Testing Guideline. (#16323) * reconstruct testing doc * Update doc/TESTS.rst.txt * DOC: add docstring for numpy.test Co-authored-by: Ross Barnowski Co-authored-by: Matti Picus --- doc/TESTS.rst.txt | 121 +++++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/doc/TESTS.rst.txt b/doc/TESTS.rst.txt index af47fe99c4db..4ab39c5869ca 100644 --- a/doc/TESTS.rst.txt +++ b/doc/TESTS.rst.txt @@ -12,7 +12,7 @@ the `pytest`_ framework. The older framework is still maintained in order to support downstream projects that use the old numpy framework, but all tests for NumPy should use pytest. -Our goal is that every module and package in SciPy and NumPy +Our goal is that every module and package in NumPy should have a thorough set of unit tests. These tests should exercise the full functionality of a given routine as well as its robustness to erroneous or unexpected input @@ -28,26 +28,30 @@ is found in a routine, you should write a new test for that specific case and add it to the test suite to prevent that bug from creeping back in unnoticed. -To run SciPy's full test suite, use the following:: +.. note:: - >>> import scipy - >>> scipy.test() + SciPy uses the testing framework from :mod:`numpy.testing`, + so all of the NumPy examples shown below are also applicable to SciPy -or from the command line:: +Testing NumPy +''''''''''''' - $ python runtests.py +NumPy can be tested in a number of ways, choose any way you feel comfortable. + +Running tests from inside Python +-------------------------------- -SciPy uses the testing framework from :mod:`numpy.testing`, so all -the SciPy examples shown here are also applicable to NumPy. NumPy's full test -suite can be run as follows:: +You can test an installed NumPy by `numpy.test`, for example, +To run NumPy's full test suite, use the following:: >>> import numpy - >>> numpy.test() + >>> numpy.test(label='slow') -The test method may take two or more arguments; the first, ``label`` is a -string specifying what should be tested and the second, ``verbose`` is an -integer giving the level of output verbosity. See the docstring for -numpy.test for details. The default value for ``label`` is 'fast' - which +The test method may take two or more arguments; the first ``label`` is a +string specifying what should be tested and the second ``verbose`` is an +integer giving the level of output verbosity. See the docstring +`numpy.test` +for details. The default value for ``label`` is 'fast' - which will run the standard tests. The string 'full' will run the full battery of tests, including those identified as being slow to run. If ``verbose`` is 1 or less, the tests will just show information messages about the tests @@ -55,38 +59,43 @@ that are run; but if it is greater than 1, then the tests will also provide warnings on missing tests. So if you want to run every test and get messages about which modules don't have tests:: - >>> scipy.test(label='full', verbose=2) # or scipy.test('full', 2) + >>> numpy.test(label='full', verbose=2) # or numpy.test('full', 2) -Finally, if you are only interested in testing a subset of SciPy, for -example, the ``integrate`` module, use the following:: +Finally, if you are only interested in testing a subset of NumPy, for +example, the ``core`` module, use the following:: - >>> scipy.integrate.test() + >>> numpy.core.test() -or from the command line:: +Running tests from the command line +----------------------------------- - $python runtests.py -t scipy/integrate/tests +If you want to build NumPy in order to work on NumPy itself, use +``runtests.py``.To run NumPy's full test suite:: -The rest of this page will give you a basic idea of how to add unit -tests to modules in SciPy. It is extremely important for us to have -extensive unit testing since this code is going to be used by -scientists and researchers and is being developed by a large number of -people spread across the world. So, if you are writing a package that -you'd like to become part of SciPy, please write the tests as you -develop the package. Also since much of SciPy is legacy code that was -originally written without unit tests, there are still several modules -that don't have tests yet. Please feel free to choose one of these -modules and develop tests for it as you read through -this introduction. + $ python runtests.py + +Testing a subset of NumPy:: + + $python runtests.py -t numpy/core/tests + +For detailed info on testing, see :ref:`testing-builds` + +Other methods of running tests +------------------------------ + +Run tests using your favourite IDE such as `vscode`_ or `pycharm`_ Writing your own tests '''''''''''''''''''''' -Every Python module, extension module, or subpackage in the SciPy +If you are writing a package that you'd like to become part of NumPy, +please write the tests as you develop the package. +Every Python module, extension module, or subpackage in the NumPy package directory should have a corresponding ``test_.py`` file. -Pytest examines these files for test methods (named test*) and test -classes (named Test*). +Pytest examines these files for test methods (named ``test*``) and test +classes (named ``Test*``). -Suppose you have a SciPy module ``scipy/xxx/yyy.py`` containing a +Suppose you have a NumPy module ``numpy/xxx/yyy.py`` containing a function ``zzz()``. To test this function you would create a test module called ``test_yyy.py``. If you only need to test one aspect of ``zzz``, you can simply add a test function:: @@ -100,7 +109,7 @@ a test class:: from numpy.testing import assert_, assert_raises # import xxx symbols - from scipy.xxx.yyy import zzz + from numpy.xxx.yyy import zzz class TestZzz: def test_simple(self): @@ -119,6 +128,11 @@ that makes it hard to identify the test from the output of running the test suite with ``verbose=2`` (or similar verbosity setting). Use plain comments (``#``) if necessary. +Also since much of NumPy is legacy code that was +originally written without unit tests, there are still several modules +that don't have tests yet. Please feel free to choose one of these +modules and develop tests for it. + Labeling tests -------------- @@ -126,8 +140,8 @@ As an alternative to ``pytest.mark.