From 3043864aac4942957a3e44a8c915220709da35e4 Mon Sep 17 00:00:00 2001 From: Han Genuit Date: Wed, 19 Sep 2012 15:57:12 +0200 Subject: [PATCH 1/3] BUG: Fix problems with ndindex and nditer This fixes an issue with ndindex shape tuple recognition, and an issue in the nditer where scalar input did not produce an empty index tuple. To be able to fix nditer, an extra flag has been added: NPY_ITFLAG_SCALAR and a new function NpyIter_IsScalar has been added to the nditer API. Also a few tests have been added to make sure the ndindex behaves as intended. --- numpy/core/code_generators/numpy_api.py | 1 + numpy/core/src/multiarray/nditer_api.c | 9 +++++++ numpy/core/src/multiarray/nditer_constr.c | 32 +++++++++-------------- numpy/core/src/multiarray/nditer_impl.h | 3 +++ numpy/core/src/multiarray/nditer_pywrap.c | 4 +++ numpy/lib/index_tricks.py | 3 +++ numpy/lib/tests/test_index_tricks.py | 6 +++++ 7 files changed, 39 insertions(+), 19 deletions(-) diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index c49c3c346fbf..17338934fcc2 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -328,6 +328,7 @@ 'PyDataMem_FREE': 289, 'PyDataMem_RENEW': 290, 'PyDataMem_SetEventHook': 291, + 'NpyIter_IsScalar': 292, } ufunc_types_api = { diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index 09e572f10096..b5d0731e1c6d 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -844,6 +844,15 @@ NpyIter_IsGrowInner(NpyIter *iter) return (NIT_ITFLAGS(iter)&NPY_ITFLAG_GROWINNER) != 0; } +/*NUMPY_API + * Whether the iterator output is scalar + */ +NPY_NO_EXPORT npy_bool +NpyIter_IsScalar(NpyIter *iter) +{ + return (NIT_ITFLAGS(iter)&NPY_ITFLAG_SCALAR) != 0; +} + /*NUMPY_API * Gets the size of the buffer, or 0 if buffering is not enabled */ diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 91768fa867ad..5d2c17fbbb46 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -54,8 +54,7 @@ static int npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags, char **op_dataptr, npy_uint32 *op_flags, int **op_axes, - npy_intp *itershape, - int output_scalars); + npy_intp *itershape); static void npyiter_replace_axisdata(NpyIter *iter, int iop, PyArrayObject *op, @@ -74,8 +73,7 @@ npyiter_find_best_axis_ordering(NpyIter *iter); static PyArray_Descr * npyiter_get_common_dtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, - PyArray_Descr **op_request_dtypes, - int only_inputs, int output_scalars); + PyArray_Descr **op_request_dtypes, int only_inputs); static PyArrayObject * npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, npy_uint32 flags, npyiter_opitflags *op_itflags, @@ -86,7 +84,7 @@ npyiter_allocate_arrays(NpyIter *iter, npy_uint32 flags, PyArray_Descr **op_dtype, PyTypeObject *subtype, npy_uint32 *op_flags, npyiter_opitflags *op_itflags, - int **op_axes, int output_scalars); + int **op_axes); static void npyiter_get_priority_subtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, @@ -123,7 +121,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, npy_int8 *perm; NpyIter_BufferData *bufferdata = NULL; int any_allocate = 0, any_missing_dtypes = 0, - output_scalars = 0, need_subtype = 0; + need_subtype = 0; /* The subtype for automatically allocated outputs */ double subtype_priority = NPY_PRIORITY; @@ -177,7 +175,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, /* If 'ndim' is zero, any outputs should be scalars */ if (ndim == 0) { - output_scalars = 1; + itflags |= NPY_ITFLAG_SCALAR; ndim = 1; } @@ -231,8 +229,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, /* Fill in the AXISDATA arrays and set the ITERSIZE field */ if (!npyiter_fill_axisdata(iter, flags, op_itflags, op_dataptr, - op_flags, op_axes, itershape, - output_scalars)) { + op_flags, op_axes, itershape)) { NpyIter_Deallocate(iter); return NULL; } @@ -338,8 +335,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, dtype = npyiter_get_common_dtype(nop, op, op_itflags, op_dtype, op_request_dtypes, - only_inputs, - output_scalars); + only_inputs); if (dtype == NULL) { NpyIter_Deallocate(iter); return NULL; @@ -389,7 +385,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, * done now using a memory layout matching the iterator. */ if (!npyiter_allocate_arrays(iter, flags, op_dtype, subtype, op_flags, - op_itflags, op_axes, output_scalars)) { + op_itflags, op_axes)) { NpyIter_Deallocate(iter); return NULL; } @@ -1437,8 +1433,7 @@ static int npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags, char **op_dataptr, npy_uint32 *op_flags, int **op_axes, - npy_intp *itershape, - int output_scalars) + npy_intp *itershape) { npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); @@ -1558,7 +1553,7 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf ondim = PyArray_NDIM(op_cur); if (bshape == 1) { strides[iop] = 0; - if (idim >= ondim && !output_scalars && + if (idim >= ondim && !(itflags & NPY_ITFLAG_SCALAR) && (op_flags[iop] & NPY_ITER_NO_BROADCAST)) { goto operand_different_than_broadcast; } @@ -2393,8 +2388,7 @@ npyiter_find_best_axis_ordering(NpyIter *iter) static PyArray_Descr * npyiter_get_common_dtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, - PyArray_Descr **op_request_dtypes, - int only_inputs, int output_scalars) + PyArray_Descr **op_request_dtypes, int only_inputs) { int iop; npy_intp narrs = 0, ndtypes = 0; @@ -2693,7 +2687,7 @@ npyiter_allocate_arrays(NpyIter *iter, npy_uint32 flags, PyArray_Descr **op_dtype, PyTypeObject *subtype, npy_uint32 *op_flags, npyiter_opitflags *op_itflags, - int **op_axes, int output_scalars) + int **op_axes) { npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); @@ -2724,7 +2718,7 @@ npyiter_allocate_arrays(NpyIter *iter, if (op[iop] == NULL) { PyArrayObject *out; PyTypeObject *op_subtype; - int ondim = output_scalars ? 0 : ndim; + int ondim = (itflags & NPY_ITFLAG_SCALAR) ? 0 : ndim; /* Check whether the subtype was disabled */ op_subtype = (op_flags[iop] & NPY_ITER_NO_SUBTYPE) ? diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index 1251baa6eada..3958dfaebe61 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -101,6 +101,8 @@ #define NPY_ITFLAG_REDUCE 0x1000 /* Reduce iteration doesn't need to recalculate reduce loops next time */ #define NPY_ITFLAG_REUSE_REDUCE_LOOPS 0x2000 +/* The iterator output is scalar */ +#define NPY_ITFLAG_SCALAR 0x4000 /* Internal iterator per-operand iterator flags */ @@ -215,6 +217,7 @@ typedef npy_int16 npyiter_opitflags; &(iter)->iter_flexdata + NIT_RESETDATAPTR_OFFSET(itflags, ndim, nop))) #define NIT_BASEOFFSETS(iter) ((npy_intp *)( \ &(iter)->iter_flexdata + NIT_BASEOFFSETS_OFFSET(itflags, ndim, nop))) + #define NIT_OPERANDS(iter) ((PyArrayObject **)( \ &(iter)->iter_flexdata + NIT_OPERANDS_OFFSET(itflags, ndim, nop))) #define NIT_OPITFLAGS(iter) ((npyiter_opitflags *)( \ diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index fd88cdbc7f87..c4e4e268d4b5 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -1542,6 +1542,9 @@ static PyObject *npyiter_multi_index_get(NewNpyArrayIterObject *self) } if (self->get_multi_index != NULL) { + if (NpyIter_IsScalar(self->iter)) { + return PyTuple_New(0); + } ndim = NpyIter_GetNDim(self->iter); self->get_multi_index(self->iter, multi_index); ret = PyTuple_New(ndim); @@ -1968,6 +1971,7 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) return NULL; } + if (NpyIter_HasDelayedBufAlloc(self->iter)) { PyErr_SetString(PyExc_ValueError, "Iterator construction used delayed buffer allocation, " diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index b07fde27db46..cc4f1240ac46 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -533,6 +533,9 @@ class ndindex(object): """ def __init__(self, *shape): + # Accept shapes in the form f(x, y, ..) as well as f((x ,y, ..)) + if len(shape) == 1 and isinstance(shape[0], tuple): + shape = shape[0] x = as_strided(_nx.zeros(1), shape=shape, strides=_nx.zeros_like(shape)) self._it = _nx.nditer(x, flags=['multi_index'], order='C') diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 0ede40d5a337..ea9a37011516 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -241,6 +241,12 @@ def test_ndindex(): x = list(np.ndindex(1, 2, 3)) expected = [ix for ix, e in np.ndenumerate(np.zeros((1, 2, 3)))] assert_array_equal(x, expected) + # Packed as well as unpacked tuple are acceptable + y = list(np.ndindex((1, 2, 3))) + assert_array_equal(x, y) + # Empty shape gives empty index + z = list(np.ndindex(())) + assert_equal(z, [()]) if __name__ == "__main__": From 71c7babb3e9d42d90cbb1c3fa516f71d0879d72b Mon Sep 17 00:00:00 2001 From: Han Genuit Date: Wed, 19 Sep 2012 17:00:30 +0200 Subject: [PATCH 2/3] STY: Remove unnecessary whitespace changes --- numpy/core/src/multiarray/nditer_impl.h | 1 - numpy/core/src/multiarray/nditer_pywrap.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index 3958dfaebe61..172be1fea1e8 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -217,7 +217,6 @@ typedef npy_int16 npyiter_opitflags; &(iter)->iter_flexdata + NIT_RESETDATAPTR_OFFSET(itflags, ndim, nop))) #define NIT_BASEOFFSETS(iter) ((npy_intp *)( \ &(iter)->iter_flexdata + NIT_BASEOFFSETS_OFFSET(itflags, ndim, nop))) - #define NIT_OPERANDS(iter) ((PyArrayObject **)( \ &(iter)->iter_flexdata + NIT_OPERANDS_OFFSET(itflags, ndim, nop))) #define NIT_OPITFLAGS(iter) ((npyiter_opitflags *)( \ diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index c4e4e268d4b5..7534b9fc3e36 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -1812,7 +1812,6 @@ static PyObject *npyiter_has_delayed_bufalloc_get(NewNpyArrayIterObject *self) "Iterator is invalid"); return NULL; } - if (NpyIter_HasDelayedBufAlloc(self->iter)) { Py_RETURN_TRUE; } @@ -1971,7 +1970,6 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) return NULL; } - if (NpyIter_HasDelayedBufAlloc(self->iter)) { PyErr_SetString(PyExc_ValueError, "Iterator construction used delayed buffer allocation, " From cfecadaeb3d4181805c567af32fd0061dc6726e4 Mon Sep 17 00:00:00 2001 From: Han Genuit Date: Thu, 20 Sep 2012 18:26:49 +0200 Subject: [PATCH 3/3] API: Remove NpyIter_IsScalar from external API --- numpy/core/code_generators/numpy_api.py | 1 - numpy/core/src/multiarray/nditer_api.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 17338934fcc2..c49c3c346fbf 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -328,7 +328,6 @@ 'PyDataMem_FREE': 289, 'PyDataMem_RENEW': 290, 'PyDataMem_SetEventHook': 291, - 'NpyIter_IsScalar': 292, } ufunc_types_api = { diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index b5d0731e1c6d..ae8a81775f3b 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -844,7 +844,7 @@ NpyIter_IsGrowInner(NpyIter *iter) return (NIT_ITFLAGS(iter)&NPY_ITFLAG_GROWINNER) != 0; } -/*NUMPY_API +/* * Whether the iterator output is scalar */ NPY_NO_EXPORT npy_bool