Skip to content

Commit 3626d0c

Browse files
njsmithnjsmith
committed
Remove PyArray_ReduceWrapper from public API
There are two reasons to want to keep PyArray_ReduceWrapper out of the public multiarray API: - Its signature is likely to change if/when masked arrays are added - It is essentially a wrapper for array->scalar transformations (*not* just reductions as its name implies -- the whole reason it is in multiarray.so in the first place is to support count_nonzero, which is not actually a reduction!). It provides some nice conveniences (like making it easy to apply such functions to multiple axes simultaneously), but, we already have a general mechanism for writing array->scalar transformations -- generalized ufuncs. We do not want to have two independent, redundant implementations of this functionality, one in multiarray and one in umath! So in the long run we should add these nice features to the generalized ufunc machinery. And in the short run, we shouldn't add it to the public API and commit ourselves to supporting it. However, simply removing it from numpy_api.py is not easy, because this code was used in both multiarray and umath. This commit: - Moves ReduceWrapper and supporting code to umath/, and makes appropriate changes (e.g. renaming it to PyUFunc_ReduceWrapper and cleaning up the header files). - Reverts numpy.count_nonzero to its previous implementation, so that it loses the new axis= and keepdims= arguments. This is unfortunate, but this change isn't so urgent that it's worth tying our APIs in knots forever. (Perhaps in the future it can become a generalized ufunc.)
1 parent 605c2b4 commit 3626d0c

16 files changed

+253
-355
lines changed

‎numpy/add_newdocs.py

-13
Original file line numberDiff line numberDiff line change
@@ -901,14 +901,6 @@ def luf(lamdaexpr, *args, **kwargs):
901901
----------
902902
a : array_like
903903
The array for which to count non-zeros.
904-
axis : None or int or tuple of ints, optional
905-
Axis or axes along which a reduction is performed.
906-
The default (`axis` = None) is perform a reduction over all
907-
the dimensions of the input array.
908-
keepdims : bool, optional
909-
If this is set to True, the axes which are reduced are left
910-
in the result as dimensions with size one. With this option,
911-
the result will broadcast correctly against the original `arr`.
912904
913905
Returns
914906
-------
@@ -925,11 +917,6 @@ def luf(lamdaexpr, *args, **kwargs):
925917
4
926918
>>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]])
927919
5
928-
>>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=1)
929-
array([2, 3])
930-
>>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=1, keepdims=True)
931-
array([[2],
932-
[3]])
933920
""")
934921

935922
add_newdoc('numpy.core.multiarray','set_typeDict',

‎numpy/core/SConscript

+1-1
Original file line numberDiff line numberDiff line change
@@ -469,7 +469,6 @@ if ENABLE_SEPARATE_COMPILATION:
469469
pjoin('src', 'multiarray', 'item_selection.c'),
470470
pjoin('src', 'multiarray', 'calculation.c'),
471471
pjoin('src', 'multiarray', 'common.c'),
472-
pjoin('src', 'multiarray', 'reduction.c'),
473472
pjoin('src', 'multiarray', 'refcount.c'),
474473
pjoin('src', 'multiarray', 'conversion_utils.c'),
475474
pjoin('src', 'multiarray', 'usertypes.c'),
@@ -498,6 +497,7 @@ env.DistutilsPythonExtension('multiarray_tests', source=multiarray_tests_src)
498497
if ENABLE_SEPARATE_COMPILATION:
499498
umathmodule_src.extend([pjoin('src', 'umath', 'ufunc_object.c')])
500499
umathmodule_src.extend([pjoin('src', 'umath', 'ufunc_type_resolution.c')])
500+
umathmodule_src.extend([pjoin('src', 'multiarray', 'reduction.c')]),
501501
umathmodule_src.extend(umath_loops_src)
502502
else:
503503
umathmodule_src = [pjoin('src', 'umath', 'umathmodule_onefile.c')]

‎numpy/core/code_generators/cversions.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,5 @@
99
# Version 6 (NumPy 1.6) added new iterator, half float and casting functions,
1010
# PyArray_CountNonzero, PyArray_NewLikeArray and PyArray_MatrixProduct2.
1111
0x00000006 = e61d5dc51fa1c6459328266e215d6987
12-
# Version 7 (NumPy 1.7) added API for NA, improved datetime64, misc utilities.
13-
0x00000007 = 280023b3ecfc2ad0326874917f6f16f9
12+
# Version 7 (NumPy 1.7) improved datetime64, misc utilities.
13+
0x00000007 = 1768b6c404a3d5a2a6bfe7c68f89e3aa

‎numpy/core/code_generators/genapi.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@
4848
join('multiarray', 'nditer_pywrap.c'),
4949
join('multiarray', 'nditer_templ.c.src'),
5050
join('multiarray', 'number.c'),
51-
join('multiarray', 'reduction.c'),
5251
join('multiarray', 'refcount.c'),
5352
join('multiarray', 'scalartypes.c.src'),
5453
join('multiarray', 'scalarapi.c'),
@@ -58,6 +57,7 @@
5857
join('umath', 'loops.c.src'),
5958
join('umath', 'ufunc_object.c'),
6059
join('umath', 'ufunc_type_resolution.c'),
60+
join('umath', 'reduction.c'),
6161
]
6262
THIS_DIR = os.path.dirname(__file__)
6363
API_FILES = [os.path.join(THIS_DIR, '..', 'src', a) for a in API_FILES]

‎numpy/core/code_generators/numpy_api.py

+5-6
Original file line numberDiff line numberDiff line change
@@ -319,12 +319,11 @@
319319
# End 1.6 API
320320
'NpyIter_IsFirstVisit': 281,
321321
'PyArray_SetBaseObject': 282,
322-
'PyArray_ReduceWrapper': 283,
323-
'PyArray_CreateSortedStridePerm': 284,
324-
'PyArray_RemoveAxesInPlace': 285,
325-
'PyArray_DebugPrint': 286,
326-
'PyArray_FailUnlessWriteable': 287,
327-
'PyArray_SetUpdateIfCopyBase': 288,
322+
'PyArray_CreateSortedStridePerm': 283,
323+
'PyArray_RemoveAxesInPlace': 284,
324+
'PyArray_DebugPrint': 285,
325+
'PyArray_FailUnlessWriteable': 286,
326+
'PyArray_SetUpdateIfCopyBase': 287,
328327
}
329328

330329
ufunc_types_api = {

‎numpy/core/include/numpy/ndarraytypes.h

-106
Original file line numberDiff line numberDiff line change
@@ -1669,112 +1669,6 @@ typedef struct {
16691669
npy_intp perm, stride;
16701670
} npy_stride_sort_item;
16711671

1672-
/************************************************************
1673-
* Typedefs used by PyArray_ReduceWrapper, new in 1.7.
1674-
************************************************************/
1675-
1676-
/*
1677-
* This is a function for assigning a reduction identity to the result,
1678-
* before doing the reduction computation. The
1679-
* value in 'data' is passed through from PyArray_ReduceWrapper.
1680-
*
1681-
* This function could, for example, simply be a call like
1682-
* return PyArray_AssignZero(result, NULL, NULL);
1683-
*
1684-
* It should return -1 on failure, or 0 on success.
1685-
*/
1686-
typedef int (PyArray_AssignReduceIdentityFunc)(PyArrayObject *result,
1687-
void *data);
1688-
1689-
/*
1690-
* This is a function for the reduce loop.
1691-
*
1692-
* The needs_api parameter indicates whether it's ok to release the GIL during
1693-
* the loop, such as when the iternext() function never calls
1694-
* a function which could raise a Python exception.
1695-
*
1696-
* Ths skip_first_count parameter indicates how many elements need to be
1697-
* skipped based on NpyIter_IsFirstVisit checks. This can only be positive
1698-
* when the 'assign_identity' parameter was NULL when calling
1699-
* PyArray_ReduceWrapper.
1700-
*
1701-
* The loop gets two data pointers and two strides, and should
1702-
* look roughly like this:
1703-
* {
1704-
* NPY_BEGIN_THREADS_DEF;
1705-
* if (!needs_api) {
1706-
* NPY_BEGIN_THREADS;
1707-
* }
1708-
* // This first-visit loop can be skipped if 'assign_identity' was non-NULL
1709-
* if (skip_first_count > 0) {
1710-
* do {
1711-
* char *data0 = dataptr[0], *data1 = dataptr[1];
1712-
* npy_intp stride0 = strideptr[0], stride1 = strideptr[1];
1713-
* npy_intp count = *countptr;
1714-
*
1715-
* // Skip any first-visit elements
1716-
* if (NpyIter_IsFirstVisit(iter, 0)) {
1717-
* if (stride0 == 0) {
1718-
* --count;
1719-
* --skip_first_count;
1720-
* data1 += stride1;
1721-
* }
1722-
* else {
1723-
* skip_first_count -= count;
1724-
* count = 0;
1725-
* }
1726-
* }
1727-
*
1728-
* while (count--) {
1729-
* *(result_t *)data0 = my_reduce_op(*(result_t *)data0,
1730-
* *(operand_t *)data1);
1731-
* data0 += stride0;
1732-
* data1 += stride1;
1733-
* }
1734-
*
1735-
* // Jump to the faster loop when skipping is done
1736-
* if (skip_first_count == 0) {
1737-
* if (iternext(iter)) {
1738-
* break;
1739-
* }
1740-
* else {
1741-
* goto finish_loop;
1742-
* }
1743-
* }
1744-
* } while (iternext(iter));
1745-
* }
1746-
* do {
1747-
* char *data0 = dataptr[0], *data1 = dataptr[1];
1748-
* npy_intp stride0 = strideptr[0], stride1 = strideptr[1];
1749-
* npy_intp count = *countptr;
1750-
*
1751-
* while (count--) {
1752-
* *(result_t *)data0 = my_reduce_op(*(result_t *)data0,
1753-
* *(operand_t *)data1);
1754-
* data0 += stride0;
1755-
* data1 += stride1;
1756-
* }
1757-
* } while (iternext(iter));
1758-
* finish_loop:
1759-
* if (!needs_api) {
1760-
* NPY_END_THREADS;
1761-
* }
1762-
* return (needs_api && PyErr_Occurred()) ? -1 : 0;
1763-
* }
1764-
*
1765-
* If needs_api is True, this function should call PyErr_Occurred()
1766-
* to check if an error occurred during processing, and return -1 for
1767-
* error, 0 for success.
1768-
*/
1769-
typedef int (PyArray_ReduceLoopFunc)(NpyIter *iter,
1770-
char **dataptr,
1771-
npy_intp *strideptr,
1772-
npy_intp *countptr,
1773-
NpyIter_IterNextFunc *iternext,
1774-
int needs_api,
1775-
npy_intp skip_first_count,
1776-
void *data);
1777-
17781672
/************************************************************
17791673
* This is the form of the struct that's returned pointed by the
17801674
* PyCObject attribute of an array __array_struct__. See

‎numpy/core/setup.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -719,7 +719,6 @@ def generate_multiarray_templated_sources(ext, build_dir):
719719
join('src', 'multiarray', 'numpymemoryview.h'),
720720
join('src', 'multiarray', 'number.h'),
721721
join('src', 'multiarray', 'numpyos.h'),
722-
join('src', 'multiarray', 'reduction.h'),
723722
join('src', 'multiarray', 'refcount.h'),
724723
join('src', 'multiarray', 'scalartypes.h'),
725724
join('src', 'multiarray', 'sequence.h'),
@@ -784,7 +783,6 @@ def generate_multiarray_templated_sources(ext, build_dir):
784783
join('src', 'multiarray', 'number.c'),
785784
join('src', 'multiarray', 'numpymemoryview.c'),
786785
join('src', 'multiarray', 'numpyos.c'),
787-
join('src', 'multiarray', 'reduction.c'),
788786
join('src', 'multiarray', 'refcount.c'),
789787
join('src', 'multiarray', 'sequence.c'),
790788
join('src', 'multiarray', 'shape.c'),
@@ -847,6 +845,7 @@ def generate_umath_c(ext, build_dir):
847845

848846
umath_src = [
849847
join('src', 'umath', 'umathmodule.c'),
848+
join('src', 'umath', 'reduction.c'),
850849
join('src', 'umath', 'funcs.inc.src'),
851850
join('src', 'umath', 'loops.c.src'),
852851
join('src', 'umath', 'ufunc_object.c'),

‎numpy/core/src/multiarray/item_selection.c

-94
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
#include "arrayobject.h"
1818
#include "ctors.h"
1919
#include "lowlevel_strided_loops.h"
20-
#include "reduction.h"
2120

2221
#include "item_selection.h"
2322

@@ -1921,99 +1920,6 @@ count_boolean_trues(int ndim, char *data, npy_intp *ashape, npy_intp *astrides)
19211920
return count;
19221921
}
19231922

1924-
static int
1925-
assign_reduce_identity_zero(PyArrayObject *result, void *data)
1926-
{
1927-
return PyArray_AssignZero(result, NULL);
1928-
}
1929-
1930-
static int
1931-
reduce_count_nonzero_loop(NpyIter *iter,
1932-
char **dataptr,
1933-
npy_intp *strides,
1934-
npy_intp *countptr,
1935-
NpyIter_IterNextFunc *iternext,
1936-
int needs_api,
1937-
npy_intp skip_first_count,
1938-
void *data)
1939-
{
1940-
PyArray_NonzeroFunc *nonzero = (PyArray_NonzeroFunc *)data;
1941-
PyArrayObject *arr = NpyIter_GetOperandArray(iter)[1];
1942-
1943-
NPY_BEGIN_THREADS_DEF;
1944-
1945-
if (!needs_api) {
1946-
NPY_BEGIN_THREADS;
1947-
}
1948-
1949-
/*
1950-
* 'skip_first_count' will always be 0 because we are doing a reduction
1951-
* with an identity.
1952-
*/
1953-
1954-
do {
1955-
char *data0 = dataptr[0], *data1 = dataptr[1];
1956-
npy_intp stride0 = strides[0], stride1 = strides[1];
1957-
npy_intp count = *countptr;
1958-
1959-
while (count--) {
1960-
if (nonzero(data1, arr)) {
1961-
++(*(npy_intp *)data0);
1962-
}
1963-
data0 += stride0;
1964-
data1 += stride1;
1965-
}
1966-
} while (iternext(iter));
1967-
1968-
if (!needs_api) {
1969-
NPY_END_THREADS;
1970-
}
1971-
1972-
return (needs_api && PyErr_Occurred()) ? -1 : 0;
1973-
}
1974-
1975-
/*
1976-
* A full reduction version of PyArray_CountNonzero, supporting
1977-
* an 'out' parameter and doing the count as a reduction along
1978-
* selected axes.
1979-
*/
1980-
NPY_NO_EXPORT PyObject *
1981-
PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out,
1982-
npy_bool *axis_flags, int keepdims)
1983-
{
1984-
PyArray_NonzeroFunc *nonzero;
1985-
PyArrayObject *result;
1986-
PyArray_Descr *dtype;
1987-
1988-
nonzero = PyArray_DESCR(arr)->f->nonzero;
1989-
if (nonzero == NULL) {
1990-
PyErr_SetString(PyExc_TypeError,
1991-
"Cannot count the number of non-zeros for a dtype "
1992-
"which doesn't have a 'nonzero' function");
1993-
return NULL;
1994-
}
1995-
1996-
dtype = PyArray_DescrFromType(NPY_INTP);
1997-
if (dtype == NULL) {
1998-
return NULL;
1999-
}
2000-
2001-
result = PyArray_ReduceWrapper(arr, out, NULL,
2002-
PyArray_DESCR(arr), dtype,
2003-
NPY_SAME_KIND_CASTING,
2004-
axis_flags, 1, keepdims, 0,
2005-
&assign_reduce_identity_zero,
2006-
&reduce_count_nonzero_loop,
2007-
nonzero, 0, "count_nonzero");
2008-
Py_DECREF(dtype);
2009-
if (out == NULL && result != NULL) {
2010-
return PyArray_Return(result);
2011-
}
2012-
else {
2013-
return (PyObject *)result;
2014-
}
2015-
}
2016-
20171923
/*NUMPY_API
20181924
* Counts the number of non-zero elements in the array.
20191925
*

‎numpy/core/src/multiarray/item_selection.h

-11
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,4 @@ NPY_NO_EXPORT int
2727
PyArray_MultiIndexSetItem(PyArrayObject *self, npy_intp *multi_index,
2828
PyObject *obj);
2929

30-
/*
31-
* A full reduction version of PyArray_CountNonzero, supporting
32-
* an 'out' parameter and doing the count as a reduction along
33-
* selected axes.
34-
*/
35-
NPY_NO_EXPORT PyObject *
36-
PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out,
37-
npy_bool *axis_flags, int keepdims);
38-
39-
40-
4130
#endif

0 commit comments

Comments
 (0)