From 4fd7e8463e5369e0754bb31dabe9e40457672e1a Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 6 Nov 2016 10:21:31 -0700 Subject: [PATCH 01/43] ENH: Revert "Temporarily disable __numpy_ufunc__" This reverts commit bac094caf14e420a801cf952080aa443a3865d97 and enables __numpy_ufunc__ for development in the NumPy 1.13.0 development cycle. --- numpy/core/src/private/ufunc_override.h | 6 ------ numpy/core/tests/test_multiarray.py | 20 -------------------- numpy/core/tests/test_umath.py | 25 ------------------------- 3 files changed, 51 deletions(-) diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 59a90c770542..4042eae2fde2 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -198,12 +198,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Pos of each override in args */ int with_override_pos[NPY_MAXARGS]; - /* 2016-01-29: Disable for now in master -- can re-enable once details are - * sorted out. All commented bits are tagged NUMPY_UFUNC_DISABLED. -njs - */ - result = NULL; - return 0; - /* * Check inputs */ diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 82b8fec14c5e..e30f35856b2b 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2407,9 +2407,6 @@ def test_dot(self): assert_equal(c, np.dot(a, b)) def test_dot_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - class A(object): def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return "A" @@ -2887,9 +2884,6 @@ def test_elide_scalar(self): assert_(type(~(a & a)) is np.bool_) def test_ufunc_override_rop_precedence(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # Check that __rmul__ and other right-hand operations have # precedence over __numpy_ufunc__ @@ -3008,9 +3002,6 @@ def __rop__(self, *other): yield check, op_name, False def test_ufunc_override_rop_simple(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # Check parts of the binary op overriding behavior in an # explicit test case that is easier to understand. class SomeClass(object): @@ -3115,9 +3106,6 @@ def __rsub__(self, other): assert_(isinstance(res, SomeClass3)) def test_ufunc_override_normalize_signature(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # gh-5674 class SomeClass(object): def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): @@ -3134,9 +3122,6 @@ def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): assert_equal(kw['signature'], 'ii->i') def test_numpy_ufunc_index(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) class CheckIndex(object): @@ -3174,9 +3159,6 @@ def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): assert_equal(np.add(a, dummy, out=a), 0) def test_out_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return - # regression test for github bug 4753 class OutClass(np.ndarray): def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): @@ -5320,8 +5302,6 @@ def test_matrix_matrix_values(self): assert_equal(res, tgt12_21) def test_numpy_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(np.ndarray): def __new__(cls, *args, **kwargs): diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index d3b379a524fa..7118808a65c6 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1568,24 +1568,7 @@ def __array__(self): assert_equal(ncu.maximum(a, B()), 0) assert_equal(ncu.maximum(a, C()), 0) - def test_ufunc_override_disabled(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - # This test should be removed when __numpy_ufunc__ is re-enabled. - - class MyArray(object): - def __numpy_ufunc__(self, *args, **kwargs): - self._numpy_ufunc_called = True - - my_array = MyArray() - real_array = np.ones(10) - assert_raises(TypeError, lambda: real_array + my_array) - assert_raises(TypeError, np.add, real_array, my_array) - assert not hasattr(my_array, "_numpy_ufunc_called") - - def test_ufunc_override(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): @@ -1611,8 +1594,6 @@ def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): assert_equal(res1[5], {}) def test_ufunc_override_mro(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return # Some multi arg functions for testing. def tres_mul(a, b, c): @@ -1704,8 +1685,6 @@ def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): assert_raises(TypeError, four_mul_ufunc, 1, c, c_sub, c) def test_ufunc_override_methods(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): @@ -1810,8 +1789,6 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[4], (a, [4, 2], 'b0')) def test_ufunc_override_out(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): @@ -1846,8 +1823,6 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res7['out'][1], 'out1') def test_ufunc_override_exception(self): - # 2016-01-29: NUMPY_UFUNC_DISABLED - return class A(object): def __numpy_ufunc__(self, *a, **kwargs): From fcd11d2b0e30a3eb1ca8d391ee965431c4a1fdfd Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 8 Nov 2016 18:16:30 -0700 Subject: [PATCH 02/43] ENH: Rename __numpy_ufunc__ to __array_ufunc__. The first commit in changing __numpy_ufunc__ by removing the index argument and making the out argument value always a tuple. These changes were proposed in gh-5986 and have been accepted. Renaming before further changes avoids triggering tests in scipy and astropy while keeping the numpy tests working. --- doc/source/reference/arrays.classes.rst | 22 ++++----- numpy/core/src/multiarray/cblasfuncs.c | 2 +- numpy/core/src/multiarray/multiarraymodule.c | 2 +- numpy/core/src/multiarray/number.c | 6 +-- numpy/core/src/private/ufunc_override.h | 14 +++--- numpy/core/src/umath/umathmodule.c | 2 +- numpy/core/tests/test_multiarray.py | 50 ++++++++++---------- numpy/core/tests/test_ufunc.py | 2 +- numpy/core/tests/test_umath.py | 20 ++++---- numpy/ma/core.py | 2 +- 10 files changed, 61 insertions(+), 61 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 298e81717578..cfd5f462e1fe 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -39,7 +39,7 @@ Special attributes and methods NumPy provides several hooks that classes can customize: -.. method:: class.__numpy_ufunc__(ufunc, method, i, inputs, **kwargs) +.. method:: class.__array_ufunc__(ufunc, method, i, inputs, **kwargs) .. versionadded:: 1.11 @@ -62,51 +62,51 @@ NumPy provides several hooks that classes can customize: :obj:`NotImplemented` if the operation requested is not implemented. - If one of the arguments has a :func:`__numpy_ufunc__` method, it is + If one of the arguments has a :func:`__array_ufunc__` method, it is executed *instead* of the ufunc. If more than one of the input - arguments implements :func:`__numpy_ufunc__`, they are tried in the + arguments implements :func:`__array_ufunc__`, they are tried in the order: subclasses before superclasses, otherwise left to right. The first routine returning something else than :obj:`NotImplemented` - determines the result. If all of the :func:`__numpy_ufunc__` + determines the result. If all of the :func:`__array_ufunc__` operations return :obj:`NotImplemented`, a :exc:`TypeError` is raised. - If an :class:`ndarray` subclass defines the :func:`__numpy_ufunc__` + If an :class:`ndarray` subclass defines the :func:`__array_ufunc__` method, this disables the :func:`__array_wrap__`, :func:`__array_prepare__`, :data:`__array_priority__` mechanism described below. - .. note:: In addition to ufuncs, :func:`__numpy_ufunc__` also + .. note:: In addition to ufuncs, :func:`__array_ufunc__` also overrides the behavior of :func:`numpy.dot` even though it is not an Ufunc. .. note:: If you also define right-hand binary operator override methods (such as ``__rmul__``) or comparison operations (such as ``__gt__``) in your class, they take precedence over the - :func:`__numpy_ufunc__` mechanism when resolving results of + :func:`__array_ufunc__` mechanism when resolving results of binary operations (such as ``ndarray_obj * your_obj``). The technical special case is: ``ndarray.__mul__`` returns ``NotImplemented`` if the other object is *not* a subclass of - :class:`ndarray`, and defines both ``__numpy_ufunc__`` and + :class:`ndarray`, and defines both ``__array_ufunc__`` and ``__rmul__``. Similar exception applies for the other operations than multiplication. In such a case, when computing a binary operation such as - ``ndarray_obj * your_obj``, your ``__numpy_ufunc__`` method + ``ndarray_obj * your_obj``, your ``__array_ufunc__`` method *will not* be called. Instead, the execution passes on to your right-hand ``__rmul__`` operation, as per standard Python operator override rules. Similar special case applies to *in-place operations*: If you define ``__rmul__``, then ``ndarray_obj *= your_obj`` *will not* - call your ``__numpy_ufunc__`` implementation. Instead, the + call your ``__array_ufunc__`` implementation. Instead, the default Python behavior ``ndarray_obj = ndarray_obj * your_obj`` occurs. Note that the above discussion applies only to Python's builtin binary operation mechanism. ``np.multiply(ndarray_obj, - your_obj)`` always calls only your ``__numpy_ufunc__``, as + your_obj)`` always calls only your ``__array_ufunc__``, as expected. .. method:: class.__array_finalize__(obj) diff --git a/numpy/core/src/multiarray/cblasfuncs.c b/numpy/core/src/multiarray/cblasfuncs.c index 4b11be947ff4..3b0b2f4f6d75 100644 --- a/numpy/core/src/multiarray/cblasfuncs.c +++ b/numpy/core/src/multiarray/cblasfuncs.c @@ -237,7 +237,7 @@ _bad_strides(PyArrayObject *ap) * This is for use by PyArray_MatrixProduct2. It is assumed on entry that * the arrays ap1 and ap2 have a common data type given by typenum that is * float, double, cfloat, or cdouble and have dimension <= 2. The - * __numpy_ufunc__ nonsense is also assumed to have been taken care of. + * __array_ufunc__ nonsense is also assumed to have been taken care of. */ NPY_NO_EXPORT PyObject * cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 2fdabb18705e..c98a8cbf0adc 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4530,7 +4530,7 @@ intern_strings(void) npy_ma_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); - npy_ma_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index f0b5637d2b13..f9d7958e9961 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -95,7 +95,7 @@ has_ufunc_attr(PyObject * obj) { return 0; } else { - return PyObject_HasAttrString(obj, "__numpy_ufunc__"); + return PyObject_HasAttrString(obj, "__array_ufunc__"); } } @@ -105,7 +105,7 @@ has_ufunc_attr(PyObject * obj) { * * This is the case when all of the following conditions apply: * - * (i) the other object defines __numpy_ufunc__ + * (i) the other object defines __array_ufunc__ * (ii) the other object defines the right-hand operation __r*__ * (iii) Python hasn't already called the right-hand operation * [occurs if the other object is a strict subclass provided @@ -118,7 +118,7 @@ has_ufunc_attr(PyObject * obj) { * This is needed, because CPython does not call __rmul__ if * the tp_number slots of the two objects are the same. * - * This always prioritizes the __r*__ routines over __numpy_ufunc__, independent + * This always prioritizes the __r*__ routines over __array_ufunc__, independent * of whether the other object is an ndarray subclass or not. */ diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 4042eae2fde2..9ba803fcea69 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -160,11 +160,11 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, } /* - * Check a set of args for the `__numpy_ufunc__` method. If more than one of - * the input arguments implements `__numpy_ufunc__`, they are tried in the + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the * order: subclasses before superclasses, otherwise left to right. The first * routine returning something other than `NotImplemented` determines the - * result. If all of the `__numpy_ufunc__` operations returns `NotImplemented`, + * result. If all of the `__array_ufunc__` operations returns `NotImplemented`, * a `TypeError` is raised. * * Returns 0 on success and 1 on exception. On success, *result contains the @@ -249,7 +249,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, _is_basic_python_type(obj)) { continue; } - if (PyObject_HasAttrString(obj, "__numpy_ufunc__")) { + if (PyObject_HasAttrString(obj, "__array_ufunc__")) { with_override[noa] = obj; with_override_pos[noa] = i; ++noa; @@ -318,7 +318,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* - * Call __numpy_ufunc__ functions in correct order + * Call __array_ufunc__ functions in correct order */ while (1) { PyObject *numpy_ufunc; @@ -361,13 +361,13 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, if (!override_obj) { /* No acceptable override found. */ PyErr_SetString(PyExc_TypeError, - "__numpy_ufunc__ not implemented for this type."); + "__array_ufunc__ not implemented for this type."); goto fail; } /* Call the override */ numpy_ufunc = PyObject_GetAttrString(override_obj, - "__numpy_ufunc__"); + "__array_ufunc__"); if (numpy_ufunc == NULL) { goto fail; } diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 2419c31f8f6f..1a6cee030c1b 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -266,7 +266,7 @@ intern_strings(void) npy_um_str_array_prepare = PyUString_InternFromString("__array_prepare__"); npy_um_str_array_wrap = PyUString_InternFromString("__array_wrap__"); npy_um_str_array_finalize = PyUString_InternFromString("__array_finalize__"); - npy_um_str_ufunc = PyUString_InternFromString("__numpy_ufunc__"); + npy_um_str_ufunc = PyUString_InternFromString("__array_ufunc__"); npy_um_str_pyvals_name = PyUString_InternFromString(UFUNC_PYVALS_NAME); return npy_um_str_out && npy_um_str_subok && npy_um_str_array_prepare && diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index e30f35856b2b..742e7014cbc0 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2408,11 +2408,11 @@ def test_dot(self): def test_dot_override(self): class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return "A" class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return NotImplemented a = A() @@ -2885,7 +2885,7 @@ def test_elide_scalar(self): def test_ufunc_override_rop_precedence(self): # Check that __rmul__ and other right-hand operations have - # precedence over __numpy_ufunc__ + # precedence over __array_ufunc__ ops = { '__add__': ('__radd__', np.add, True), @@ -2913,8 +2913,8 @@ class OtherNdarraySubclass(np.ndarray): pass class OtherNdarraySubclassWithOverride(np.ndarray): - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " + def __array_ufunc__(self, *a, **kw): + raise AssertionError(("__array_ufunc__ %r %r shouldn't have " "been called!") % (a, kw)) def check(op_name, ndsubclass): @@ -2933,8 +2933,8 @@ def check(op_name, ndsubclass): def __init__(self, *a, **kw): pass - def __numpy_ufunc__(self, *a, **kw): - raise AssertionError(("__numpy_ufunc__ %r %r shouldn't have " + def __array_ufunc__(self, *a, **kw): + raise AssertionError(("__array_ufunc__ %r %r shouldn't have " "been called!") % (a, kw)) def __op__(self, *other): @@ -2949,7 +2949,7 @@ def __rop__(self, *other): bases = (object,) dct = {'__init__': __init__, - '__numpy_ufunc__': __numpy_ufunc__, + '__array_ufunc__': __array_ufunc__, op_name: __op__} if op_name != rop_name: dct[rop_name] = __rop__ @@ -2989,7 +2989,7 @@ def __rop__(self, *other): # integer-like? assert_equal(iop(arr, obj), "rop", err_msg=err_msg) - # Check that ufunc call __numpy_ufunc__ normally + # Check that ufunc call __array_ufunc__ normally if np_op is not None: assert_raises(AssertionError, np_op, arr, obj, err_msg=err_msg) @@ -3005,7 +3005,7 @@ def test_ufunc_override_rop_simple(self): # Check parts of the binary op overriding behavior in an # explicit test case that is easier to understand. class SomeClass(object): - def __numpy_ufunc__(self, *a, **kw): + def __array_ufunc__(self, *a, **kw): return "ufunc" def __mul__(self, other): @@ -3024,7 +3024,7 @@ def __lt__(self, other): return "nope" class SomeClass2(SomeClass, np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, i, inputs, **kw): if ufunc is np.multiply or ufunc is np.bitwise_and: return "ufunc" else: @@ -3052,17 +3052,17 @@ def __rsub__(self, other): # obj is first, so should get to define outcome. assert_equal(obj * arr, 123) - # obj is second, but has __numpy_ufunc__ and defines __rmul__. + # obj is second, but has __array_ufunc__ and defines __rmul__. assert_equal(arr * obj, 321) - # obj is second, but has __numpy_ufunc__ and defines __rsub__. + # obj is second, but has __array_ufunc__ and defines __rsub__. assert_equal(arr - obj, "no subs for me") - # obj is second, but has __numpy_ufunc__ and defines __lt__. + # obj is second, but has __array_ufunc__ and defines __lt__. assert_equal(arr > obj, "nope") - # obj is second, but has __numpy_ufunc__ and defines __gt__. + # obj is second, but has __array_ufunc__ and defines __gt__. assert_equal(arr < obj, "yep") - # Called as a ufunc, obj.__numpy_ufunc__ is used. + # Called as a ufunc, obj.__array_ufunc__ is used. assert_equal(np.multiply(arr, obj), "ufunc") - # obj is second, but has __numpy_ufunc__ and defines __rmul__. + # obj is second, but has __array_ufunc__ and defines __rmul__. arr *= obj assert_equal(arr, 321) @@ -3072,7 +3072,7 @@ def __rsub__(self, other): assert_equal(arr - obj2, "no subs for me") assert_equal(arr > obj2, "nope") assert_equal(arr < obj2, "yep") - # Called as a ufunc, obj2.__numpy_ufunc__ is called. + # Called as a ufunc, obj2.__array_ufunc__ is called. assert_equal(np.multiply(arr, obj2), "ufunc") # Also when the method is not overridden. assert_equal(arr & obj2, "ufunc") @@ -3093,13 +3093,13 @@ def __rsub__(self, other): assert_equal(obj2 * obj3, 123) # And of course, here obj3.__mul__ should be called. assert_equal(obj3 * obj2, 123) - # obj3 defines __numpy_ufunc__ but obj3.__radd__ is obj2.__radd__. + # obj3 defines __array_ufunc__ but obj3.__radd__ is obj2.__radd__. # (and both are just ndarray.__radd__); see #4815. res = obj2 + obj3 assert_equal(res, 46) assert_(isinstance(res, SomeClass2)) # Since obj3 is a subclass, it should have precedence, like CPython - # would give, even though obj2 has __numpy_ufunc__ and __radd__. + # would give, even though obj2 has __array_ufunc__ and __radd__. # See gh-4815 and gh-5747. res = obj3 + obj2 assert_equal(res, 46) @@ -3108,7 +3108,7 @@ def __rsub__(self, other): def test_ufunc_override_normalize_signature(self): # gh-5674 class SomeClass(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, i, inputs, **kw): return kw a = SomeClass() @@ -3125,7 +3125,7 @@ def test_numpy_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) class CheckIndex(object): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, i, inputs, **kw): return i a = CheckIndex() @@ -3161,7 +3161,7 @@ def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): def test_out_override(self): # regression test for github bug 4753 class OutClass(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, i, inputs, **kw): if 'out' in kw: tmp_kw = kw.copy() tmp_kw.pop('out') @@ -5307,14 +5307,14 @@ class A(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return "A" class B(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return NotImplemented a = A([1, 2]) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 26e15539eb0e..e470a4169143 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1225,7 +1225,7 @@ def test_structured_equal(self): # https://github.com/numpy/numpy/issues/4855 class MyA(np.ndarray): - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, i, inputs, **kwargs): return getattr(ufunc, method)(*(input.view(np.ndarray) for input in inputs), **kwargs) a = np.arange(12.).reshape(4,3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 7118808a65c6..c5b7f862f01f 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1571,7 +1571,7 @@ def __array__(self): def test_ufunc_override(self): class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return self, func, method, pos, inputs, kwargs a = A() @@ -1607,23 +1607,23 @@ def quatro_mul(a, b, c, d): four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) class A(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return "A" class ASub(A): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return "ASub" class B(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return "B" class C(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return NotImplemented class CSub(object): - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): return NotImplemented a = A() @@ -1687,7 +1687,7 @@ def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): def test_ufunc_override_methods(self): class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return self, ufunc, method, pos, inputs, kwargs # __call__ @@ -1791,11 +1791,11 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): def test_ufunc_override_out(self): class A(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return kwargs class B(object): - def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): return kwargs a = A() @@ -1825,7 +1825,7 @@ def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): def test_ufunc_override_exception(self): class A(object): - def __numpy_ufunc__(self, *a, **kwargs): + def __array_ufunc__(self, *a, **kwargs): raise ValueError("oops") a = A() diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 554fd6dc517e..c935edc78760 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -3935,7 +3935,7 @@ def _delegate_binop(self, other): # This emulates the logic in # multiarray/number.c:PyArray_GenericBinaryFunction if (not isinstance(other, np.ndarray) - and not hasattr(other, "__numpy_ufunc__")): + and not hasattr(other, "__array_ufunc__")): other_priority = getattr(other, "__array_priority__", -1000000) if self.__array_priority__ < other_priority: return True From c7b25e26bb52cc8c75026bf441dcdcd6a2ef6085 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sat, 12 Nov 2016 12:32:46 -0700 Subject: [PATCH 03/43] ENH: Remove position arg from __array_ufunc__. Previously when __array_ufunc__ for one of the ufunc arguments was called, that arguments position was passed in the call. This PR removes that argument as proposed in gh-5986. --- doc/source/reference/arrays.classes.rst | 3 +- numpy/core/src/private/ufunc_override.h | 9 +-- numpy/core/tests/test_multiarray.py | 56 +++++++++--------- numpy/core/tests/test_ufunc.py | 2 +- numpy/core/tests/test_umath.py | 75 +++++++++++-------------- 5 files changed, 64 insertions(+), 81 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index cfd5f462e1fe..b1fb95d91684 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -39,7 +39,7 @@ Special attributes and methods NumPy provides several hooks that classes can customize: -.. method:: class.__array_ufunc__(ufunc, method, i, inputs, **kwargs) +.. method:: class.__array_ufunc__(ufunc, method, inputs, **kwargs) .. versionadded:: 1.11 @@ -51,7 +51,6 @@ NumPy provides several hooks that classes can customize: - *method* is a string indicating which Ufunc method was called (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, ``"accumulate"``, ``"outer"``, ``"inner"``). - - *i* is the index of *self* in *inputs*. - *inputs* is a tuple of the input arguments to the ``ufunc`` - *kwargs* is a dictionary containing the optional input arguments of the ufunc. The ``out`` argument is always contained in diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 9ba803fcea69..2bcaf1dec3e9 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -177,7 +177,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, int nin) { int i; - int override_pos; /* Position of override in args.*/ int j; int nargs; @@ -195,9 +194,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *with_override[NPY_MAXARGS]; - /* Pos of each override in args */ - int with_override_pos[NPY_MAXARGS]; - /* * Check inputs */ @@ -251,7 +247,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } if (PyObject_HasAttrString(obj, "__array_ufunc__")) { with_override[noa] = obj; - with_override_pos[noa] = i; ++noa; } } @@ -336,7 +331,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* Get the first instance of an overriding arg.*/ - override_pos = with_override_pos[i]; override_obj = obj; /* Check for sub-types to the right of obj. */ @@ -372,8 +366,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - override_args = Py_BuildValue("OOiO", ufunc, method_name, - override_pos, normal_args); + override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); if (override_args == NULL) { Py_DECREF(numpy_ufunc); goto fail; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 742e7014cbc0..24e7585e9687 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2408,11 +2408,11 @@ def test_dot(self): def test_dot_override(self): class A(object): - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return "A" class B(object): - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return NotImplemented a = A() @@ -3024,13 +3024,12 @@ def __lt__(self, other): return "nope" class SomeClass2(SomeClass, np.ndarray): - def __array_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, inputs, **kw): if ufunc is np.multiply or ufunc is np.bitwise_and: return "ufunc" else: - inputs = list(inputs) - if i < len(inputs): - inputs[i] = np.asarray(self) + inputs = [np.asarray(self) if i is self else i + for i in inputs] func = getattr(ufunc, method) if ('out' in kw) and (kw['out'] is not None): kw['out'] = np.asarray(kw['out']) @@ -3108,7 +3107,7 @@ def __rsub__(self, other): def test_ufunc_override_normalize_signature(self): # gh-5674 class SomeClass(object): - def __array_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, inputs, **kw): return kw a = SomeClass() @@ -3125,43 +3124,46 @@ def test_numpy_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) class CheckIndex(object): - def __array_ufunc__(self, ufunc, method, i, inputs, **kw): - return i + def __array_ufunc__(self, ufunc, method, inputs, **kw): + for i, a in enumerate(inputs): + if a is self: + return i + return None a = CheckIndex() dummy = np.arange(2.) # 1 input, 1 output assert_equal(np.sin(a), 0) - assert_equal(np.sin(dummy, a), 1) - assert_equal(np.sin(dummy, out=a), 1) - assert_equal(np.sin(dummy, out=(a,)), 1) + assert_equal(np.sin(dummy, a), None) + assert_equal(np.sin(dummy, out=a), None) + assert_equal(np.sin(dummy, out=(a,)), None) assert_equal(np.sin(a, a), 0) assert_equal(np.sin(a, out=a), 0) assert_equal(np.sin(a, out=(a,)), 0) # 1 input, 2 outputs - assert_equal(np.modf(dummy, a), 1) - assert_equal(np.modf(dummy, None, a), 2) - assert_equal(np.modf(dummy, dummy, a), 2) - assert_equal(np.modf(dummy, out=a), 1) - assert_equal(np.modf(dummy, out=(a,)), 1) - assert_equal(np.modf(dummy, out=(a, None)), 1) - assert_equal(np.modf(dummy, out=(a, dummy)), 1) - assert_equal(np.modf(dummy, out=(None, a)), 2) - assert_equal(np.modf(dummy, out=(dummy, a)), 2) + assert_equal(np.modf(dummy, a), None) + assert_equal(np.modf(dummy, None, a), None) + assert_equal(np.modf(dummy, dummy, a), None) + assert_equal(np.modf(dummy, out=a), None) + assert_equal(np.modf(dummy, out=(a,)), None) + assert_equal(np.modf(dummy, out=(a, None)), None) + assert_equal(np.modf(dummy, out=(a, dummy)), None) + assert_equal(np.modf(dummy, out=(None, a)), None) + assert_equal(np.modf(dummy, out=(dummy, a)), None) assert_equal(np.modf(a, out=(dummy, a)), 0) # 2 inputs, 1 output assert_equal(np.add(a, dummy), 0) assert_equal(np.add(dummy, a), 1) - assert_equal(np.add(dummy, dummy, a), 2) + assert_equal(np.add(dummy, dummy, a), None) assert_equal(np.add(dummy, a, a), 1) - assert_equal(np.add(dummy, dummy, out=a), 2) - assert_equal(np.add(dummy, dummy, out=(a,)), 2) + assert_equal(np.add(dummy, dummy, out=a), None) + assert_equal(np.add(dummy, dummy, out=(a,)), None) assert_equal(np.add(a, dummy, out=a), 0) def test_out_override(self): # regression test for github bug 4753 class OutClass(np.ndarray): - def __array_ufunc__(self, ufunc, method, i, inputs, **kw): + def __array_ufunc__(self, ufunc, method, inputs, **kw): if 'out' in kw: tmp_kw = kw.copy() tmp_kw.pop('out') @@ -5307,14 +5309,14 @@ class A(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return "A" class B(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return NotImplemented a = A([1, 2]) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index e470a4169143..3776db84e4f1 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1225,7 +1225,7 @@ def test_structured_equal(self): # https://github.com/numpy/numpy/issues/4855 class MyA(np.ndarray): - def __array_ufunc__(self, ufunc, method, i, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return getattr(ufunc, method)(*(input.view(np.ndarray) for input in inputs), **kwargs) a = np.arange(12.).reshape(4,3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index c5b7f862f01f..3d4f12aeb7ca 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1571,8 +1571,8 @@ def __array__(self): def test_ufunc_override(self): class A(object): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): - return self, func, method, pos, inputs, kwargs + def __array_ufunc__(self, func, method, inputs, **kwargs): + return self, func, method, inputs, kwargs a = A() b = np.matrix([1]) @@ -1586,12 +1586,10 @@ def __array_ufunc__(self, func, method, pos, inputs, **kwargs): assert_equal(res1[1], np.dot) assert_equal(res0[2], '__call__') assert_equal(res1[2], '__call__') - assert_equal(res0[3], 0) - assert_equal(res1[3], 0) - assert_equal(res0[4], (a, b)) - assert_equal(res1[4], (a, b)) - assert_equal(res0[5], {}) - assert_equal(res1[5], {}) + assert_equal(res0[3], (a, b)) + assert_equal(res1[3], (a, b)) + assert_equal(res0[4], {}) + assert_equal(res1[4], {}) def test_ufunc_override_mro(self): @@ -1607,23 +1605,23 @@ def quatro_mul(a, b, c, d): four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) class A(object): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, inputs, **kwargs): return "A" class ASub(A): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, inputs, **kwargs): return "ASub" class B(object): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, inputs, **kwargs): return "B" class C(object): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, inputs, **kwargs): return NotImplemented class CSub(object): - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, inputs, **kwargs): return NotImplemented a = A() @@ -1687,8 +1685,8 @@ def __array_ufunc__(self, func, method, pos, inputs, **kwargs): def test_ufunc_override_methods(self): class A(object): - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): - return self, ufunc, method, pos, inputs, kwargs + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + return self, ufunc, method, inputs, kwargs # __call__ a = A() @@ -1696,18 +1694,16 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], '__call__') - assert_equal(res[3], 1) - assert_equal(res[4], (1, a)) - assert_equal(res[5], {'foo': 'bar', 'answer': 42}) + assert_equal(res[3], (1, a)) + assert_equal(res[4], {'foo': 'bar', 'answer': 42}) # reduce, positional args res = np.multiply.reduce(a, 'axis0', 'dtype0', 'out0', 'keep0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'keepdims': 'keep0', 'axis': 'axis0'}) @@ -1718,9 +1714,8 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'keepdims': 'keep0', 'axis': 'axis0'}) @@ -1730,9 +1725,8 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'axis': 'axis0'}) @@ -1742,9 +1736,8 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'accumulate') - assert_equal(res[3], 0) - assert_equal(res[4], (a,)) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a,)) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'axis': 'axis0'}) @@ -1753,9 +1746,8 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'axis': 'axis0'}) @@ -1765,9 +1757,8 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduceat') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2])) - assert_equal(res[5], {'dtype':'dtype0', + assert_equal(res[3], (a, [4, 2])) + assert_equal(res[4], {'dtype':'dtype0', 'out': 'out0', 'axis': 'axis0'}) @@ -1776,26 +1767,24 @@ def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'outer') - assert_equal(res[3], 0) - assert_equal(res[4], (a, 42)) - assert_equal(res[5], {}) + assert_equal(res[3], (a, 42)) + assert_equal(res[4], {}) # at res = np.multiply.at(a, [4, 2], 'b0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'at') - assert_equal(res[3], 0) - assert_equal(res[4], (a, [4, 2], 'b0')) + assert_equal(res[3], (a, [4, 2], 'b0')) def test_ufunc_override_out(self): class A(object): - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return kwargs class B(object): - def __array_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return kwargs a = A() From 8a9e790c017983aa6985cf46b427536cf04edce5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 24 Jun 2015 00:42:50 -0700 Subject: [PATCH 04/43] MAINT: Put PyArray_GetAttrString_SuppressException in get_attr_string.h This is an ugly kluge, but until we merge multiarray.so and umath.so moving stuff into private/*.h serves as a reasonable workaround. --- numpy/core/src/multiarray/common.c | 57 +------------ numpy/core/src/multiarray/common.h | 31 ------- numpy/core/src/multiarray/ctors.c | 2 + numpy/core/src/multiarray/multiarraymodule.c | 2 + numpy/core/src/private/get_attr_string.h | 85 ++++++++++++++++++++ numpy/core/src/private/ufunc_override.h | 27 +++---- 6 files changed, 103 insertions(+), 101 deletions(-) create mode 100644 numpy/core/src/private/get_attr_string.h diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index dc9b2edec51d..1df3b8b48c41 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -14,6 +14,8 @@ #include "common.h" #include "buffer.h" +#include "get_attr_string.h" + /* * The casting to use for implicit assignment operations resulting from * in-place operations (like +=) and out= arguments. (Notice that this @@ -29,61 +31,6 @@ * warning (that people's code will be broken in a future release.) */ -/* - * PyArray_GetAttrString_SuppressException: - * - * Stripped down version of PyObject_GetAttrString, - * avoids lookups for None, tuple, and List objects, - * and doesn't create a PyErr since this code ignores it. - * - * This can be much faster then PyObject_GetAttrString where - * exceptions are not used by caller. - * - * 'obj' is the object to search for attribute. - * - * 'name' is the attribute to search for. - * - * Returns attribute value on success, 0 on failure. - */ -PyObject * -PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) -{ - PyTypeObject *tp = Py_TYPE(obj); - PyObject *res = (PyObject *)NULL; - - /* We do not need to check for special attributes on trivial types */ - if (_is_basic_python_type(obj)) { - return NULL; - } - - /* Attribute referenced by (char *)name */ - if (tp->tp_getattr != NULL) { - res = (*tp->tp_getattr)(obj, name); - if (res == NULL) { - PyErr_Clear(); - } - } - /* Attribute referenced by (PyObject *)name */ - else if (tp->tp_getattro != NULL) { -#if defined(NPY_PY3K) - PyObject *w = PyUnicode_InternFromString(name); -#else - PyObject *w = PyString_InternFromString(name); -#endif - if (w == NULL) { - return (PyObject *)NULL; - } - res = (*tp->tp_getattro)(obj, w); - Py_DECREF(w); - if (res == NULL) { - PyErr_Clear(); - } - } - return res; -} - - - NPY_NO_EXPORT NPY_CASTING NPY_DEFAULT_ASSIGN_CASTING = NPY_SAME_KIND_CASTING; diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 8da317856d06..ae9b960c86f0 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -40,9 +40,6 @@ NPY_NO_EXPORT int PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_status); -NPY_NO_EXPORT PyObject * -PyArray_GetAttrString_SuppressException(PyObject *v, char *name); - /* * Returns NULL without setting an exception if no scalar is matched, a * new dtype reference otherwise. @@ -255,34 +252,6 @@ npy_memchr(char * haystack, char needle, return p; } -static NPY_INLINE int -_is_basic_python_type(PyObject * obj) -{ - if (obj == Py_None || - PyBool_Check(obj) || - /* Basic number types */ -#if !defined(NPY_PY3K) - PyInt_CheckExact(obj) || - PyString_CheckExact(obj) || -#endif - PyLong_CheckExact(obj) || - PyFloat_CheckExact(obj) || - PyComplex_CheckExact(obj) || - /* Basic sequence types */ - PyList_CheckExact(obj) || - PyTuple_CheckExact(obj) || - PyDict_CheckExact(obj) || - PyAnySet_CheckExact(obj) || - PyUnicode_CheckExact(obj) || - PyBytes_CheckExact(obj) || - PySlice_Check(obj)) { - - return 1; - } - - return 0; -} - /* * Convert NumPy stride to BLAS stride. Returns 0 if conversion cannot be done * (BLAS won't handle negative or zero strides the way we want). diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0506d3dee3e7..f7a5f3deb4d3 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -29,6 +29,8 @@ #include "alloc.h" #include +#include "get_attr_string.h" + /* * Reading from a file or a string. * diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index c98a8cbf0adc..27e1e2af29f7 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -61,6 +61,8 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "compiled_base.h" #include "mem_overlap.h" +#include "get_attr_string.h" + /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; diff --git a/numpy/core/src/private/get_attr_string.h b/numpy/core/src/private/get_attr_string.h new file mode 100644 index 000000000000..b32be28f7a42 --- /dev/null +++ b/numpy/core/src/private/get_attr_string.h @@ -0,0 +1,85 @@ +#ifndef __GET_ATTR_STRING_H +#define __GET_ATTR_STRING_H + +static NPY_INLINE int +_is_basic_python_type(PyObject * obj) +{ + if (obj == Py_None || + PyBool_Check(obj) || + /* Basic number types */ +#if !defined(NPY_PY3K) + PyInt_CheckExact(obj) || + PyString_CheckExact(obj) || +#endif + PyLong_CheckExact(obj) || + PyFloat_CheckExact(obj) || + PyComplex_CheckExact(obj) || + /* Basic sequence types */ + PyList_CheckExact(obj) || + PyTuple_CheckExact(obj) || + PyDict_CheckExact(obj) || + PyAnySet_CheckExact(obj) || + PyUnicode_CheckExact(obj) || + PyBytes_CheckExact(obj) || + PySlice_Check(obj)) { + + return 1; + } + + return 0; +} + +/* + * PyArray_GetAttrString_SuppressException: + * + * Stripped down version of PyObject_GetAttrString, + * avoids lookups for None, tuple, and List objects, + * and doesn't create a PyErr since this code ignores it. + * + * This can be much faster then PyObject_GetAttrString where + * exceptions are not used by caller. + * + * 'obj' is the object to search for attribute. + * + * 'name' is the attribute to search for. + * + * Returns attribute value on success, 0 on failure. + */ +static PyObject * +PyArray_GetAttrString_SuppressException(PyObject *obj, char *name) +{ + PyTypeObject *tp = Py_TYPE(obj); + PyObject *res = (PyObject *)NULL; + + /* We do not need to check for special attributes on trivial types */ + if (_is_basic_python_type(obj)) { + return NULL; + } + + /* Attribute referenced by (char *)name */ + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(obj, name); + if (res == NULL) { + PyErr_Clear(); + } + } + /* Attribute referenced by (PyObject *)name */ + else if (tp->tp_getattro != NULL) { +#if defined(NPY_PY3K) + PyObject *w = PyUnicode_InternFromString(name); +#else + PyObject *w = PyString_InternFromString(name); +#endif + if (w == NULL) { + return (PyObject *)NULL; + } + res = (*tp->tp_getattro)(obj, w); + Py_DECREF(w); + if (res == NULL) { + PyErr_Clear(); + } + } + return res; +} + +#endif diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 2bcaf1dec3e9..357debf4f23a 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -6,6 +6,8 @@ #include #include "numpy/ufuncobject.h" +#include "get_attr_string.h" + static void normalize___call___args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds, @@ -184,6 +186,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, int out_kwd_is_tuple = 0; int noa = 0; /* Number of overriding args.*/ + PyObject *tmp; PyObject *obj; PyObject *out_kwd_obj = NULL; PyObject *other_obj; @@ -237,15 +240,9 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, obj = out_kwd_obj; } } - /* - * TODO: could use PyArray_GetAttrString_SuppressException if it - * weren't private to multiarray.so - */ - if (PyArray_CheckExact(obj) || PyArray_IsScalar(obj, Generic) || - _is_basic_python_type(obj)) { - continue; - } - if (PyObject_HasAttrString(obj, "__array_ufunc__")) { + tmp = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); + if (tmp) { + Py_DECREF(tmp); with_override[noa] = obj; ++noa; } @@ -316,7 +313,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, * Call __array_ufunc__ functions in correct order */ while (1) { - PyObject *numpy_ufunc; + PyObject *array_ufunc; PyObject *override_args; PyObject *override_obj; @@ -360,21 +357,21 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* Call the override */ - numpy_ufunc = PyObject_GetAttrString(override_obj, + array_ufunc = PyObject_GetAttrString(override_obj, "__array_ufunc__"); - if (numpy_ufunc == NULL) { + if (array_ufunc == NULL) { goto fail; } override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); if (override_args == NULL) { - Py_DECREF(numpy_ufunc); + Py_DECREF(array_ufunc); goto fail; } - *result = PyObject_Call(numpy_ufunc, override_args, normal_kwds); + *result = PyObject_Call(array_ufunc, override_args, normal_kwds); - Py_DECREF(numpy_ufunc); + Py_DECREF(array_ufunc); Py_DECREF(override_args); if (*result == NULL) { From 4dd538089a7341c1aa92322eebad70886fc1703e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 24 Jun 2015 14:32:49 -0700 Subject: [PATCH 05/43] MAINT: dike out a bunch of weird old code implementing scalar power For some reason, np.generic.__pow__ felt the need to reimplement Python's full binop dispatch logic. The code has been there since the beginning of (git) history, isn't required to pass any tests, and AFAICT doesn't actually do anything interesting. I suspect it was just split apart from the (much more trivial) implementation of all the other np.generic binop methods because nb_power has a slightly different signature than all the others (it takes an extra argument, which we ignore, but still has to be accepted and passed as appropriate), and then got forgotten when the other implementations were simplified long ago. This commit throws away all that code and makes gentype_power a 2-liner, just like all the other gentype_@name@ methods. --- numpy/core/src/multiarray/scalartypes.c.src | 53 +-------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 7edf3b71d88c..8908e93fc56c 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -151,63 +151,14 @@ gentype_free(PyObject *v) static PyObject * gentype_power(PyObject *m1, PyObject *m2, PyObject *modulo) { - PyObject *arr, *ret, *arg2; - char *msg="unsupported operand type(s) for ** or pow()"; - if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_IsScalar(m1, Generic)) { - if (PyArray_Check(m1)) { - ret = Py_TYPE(m1)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m2, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m2, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(m1, arr, Py_None); - Py_DECREF(arr); - } - return ret; - } - if (!PyArray_IsScalar(m2, Generic)) { - if (PyArray_Check(m2)) { - ret = Py_TYPE(m2)->tp_as_number->nb_power(m1,m2, Py_None); - } - else { - if (!PyArray_IsScalar(m1, Generic)) { - PyErr_SetString(PyExc_TypeError, msg); - return NULL; - } - arr = PyArray_FromScalar(m1, NULL); - if (arr == NULL) { - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, m2, Py_None); - Py_DECREF(arr); - } - return ret; - } - arr = arg2 = NULL; - arr = PyArray_FromScalar(m1, NULL); - arg2 = PyArray_FromScalar(m2, NULL); - if (arr == NULL || arg2 == NULL) { - Py_XDECREF(arr); - Py_XDECREF(arg2); - return NULL; - } - ret = Py_TYPE(arr)->tp_as_number->nb_power(arr, arg2, Py_None); - Py_DECREF(arr); - Py_DECREF(arg2); - return ret; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_power, gentype_power); + return PyArray_Type.tp_as_number->nb_power(m1, m2, Py_None); } static PyObject * From 7d9bc2fd597236e7490b804948856bb65f4f9d3a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 22 Jun 2015 02:55:40 -0700 Subject: [PATCH 06/43] BUG/ENH: Switch to simplified __array_ufunc__/binop interaction As per the discussion at gh-5844, and in particular https://github.com/numpy/numpy/issues/5844#issuecomment-112014014 this commit switches binop dispatch to mostly defer to ufuncs, except in some specific cases elaborated in a long comment in number.c. The basic strategy is to define a single piece of C code that knows how to handle forward binop overrides, and we put it into private/binop_override.h so that it can be accessed by both the array code in multiarray.so and the scalar code in umath.so. --- numpy/core/src/multiarray/arrayobject.c | 45 +-- numpy/core/src/multiarray/number.c | 208 ++--------- numpy/core/src/multiarray/number.h | 4 - numpy/core/src/multiarray/scalartypes.c.src | 6 +- numpy/core/src/private/binop_override.h | 292 +++++++++++++++ numpy/core/src/umath/scalarmath.c.src | 13 + numpy/core/tests/test_multiarray.py | 386 +++++++++----------- numpy/ma/core.py | 9 +- 8 files changed, 521 insertions(+), 442 deletions(-) create mode 100644 numpy/core/src/private/binop_override.h diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 8946ff2556ee..df389020128c 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -54,6 +54,8 @@ maintainer email: oliphant.travis@ieee.org #include "mem_overlap.h" #include "numpyos.h" +#include "binop_override.h" + /*NUMPY_API Compute the size of an array (in number of items) */ @@ -1335,23 +1337,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) switch (cmp_op) { case Py_LT: - if (needs_right_binop_forward(obj_self, other, "__gt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - /* See discussion in number.c */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less); break; case Py_LE: - if (needs_right_binop_forward(obj_self, other, "__ge__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyArray_GenericBinaryFunction(self, other, - n_ops.less_equal); + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); + result = PyArray_GenericBinaryFunction(self, other, n_ops.less_equal); break; case Py_EQ: /* @@ -1401,11 +1392,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__eq__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.equal); @@ -1478,11 +1465,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - if (needs_right_binop_forward(obj_self, other, "__ne__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, (PyObject *)other, n_ops.not_equal); if (result == NULL) { @@ -1502,20 +1485,12 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) } break; case Py_GT: - if (needs_right_binop_forward(obj_self, other, "__lt__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater); break; case Py_GE: - if (needs_right_binop_forward(obj_self, other, "__le__", 0) && - Py_TYPE(obj_self)->tp_richcompare != Py_TYPE(other)->tp_richcompare) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); result = PyArray_GenericBinaryFunction(self, other, n_ops.greater_equal); break; diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index f9d7958e9961..f846fb318990 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -14,6 +14,8 @@ #include "number.h" #include "temp_elide.h" +#include "binop_override.h" + /************************************************************************* **************** Implement Number Protocol **************************** *************************************************************************/ @@ -87,88 +89,6 @@ PyArray_SetNumericOps(PyObject *dict) (PyDict_SetItemString(dict, #op, n_ops.op)==-1)) \ goto fail; -static int -has_ufunc_attr(PyObject * obj) { - /* attribute check is expensive for scalar operations, avoid if possible */ - if (PyArray_CheckExact(obj) || PyArray_CheckAnyScalarExact(obj) || - _is_basic_python_type(obj)) { - return 0; - } - else { - return PyObject_HasAttrString(obj, "__array_ufunc__"); - } -} - -/* - * Check whether the operation needs to be forwarded to the right-hand binary - * operation. - * - * This is the case when all of the following conditions apply: - * - * (i) the other object defines __array_ufunc__ - * (ii) the other object defines the right-hand operation __r*__ - * (iii) Python hasn't already called the right-hand operation - * [occurs if the other object is a strict subclass provided - * the operation is not in-place] - * - * An additional check is made in GIVE_UP_IF_HAS_RIGHT_BINOP macro below: - * - * (iv) other.__class__.__r*__ is not self.__class__.__r*__ - * - * This is needed, because CPython does not call __rmul__ if - * the tp_number slots of the two objects are the same. - * - * This always prioritizes the __r*__ routines over __array_ufunc__, independent - * of whether the other object is an ndarray subclass or not. - */ - -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int inplace_op) -{ - if (other == NULL || - self == NULL || - Py_TYPE(self) == Py_TYPE(other) || - PyArray_CheckExact(other) || - PyArray_CheckAnyScalar(other)) { - /* - * Quick cases - */ - return 0; - } - if ((!inplace_op && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) || - !PyArray_Check(self)) { - /* - * Bail out if Python would already have called the right-hand - * operation. - */ - return 0; - } - if (has_ufunc_attr(other) && - PyObject_HasAttrString(other, right_name)) { - return 1; - } - else { - return 0; - } -} - -/* In pure-Python, SAME_SLOTS can be replaced by - getattr(m1, op_name) is getattr(m2, op_name) */ -#define SAME_SLOTS(m1, m2, slot_name) \ - (Py_TYPE(m1)->tp_as_number != NULL && Py_TYPE(m2)->tp_as_number != NULL && \ - Py_TYPE(m1)->tp_as_number->slot_name == Py_TYPE(m2)->tp_as_number->slot_name) - -#define GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, left_name, right_name, inplace, slot_name) \ - do { \ - if (needs_right_binop_forward((PyObject *)m1, m2, right_name, inplace) && \ - (inplace || !SAME_SLOTS(m1, m2, slot_name))) { \ - Py_INCREF(Py_NotImplemented); \ - return Py_NotImplemented; \ - } \ - } while (0) - - /*NUMPY_API Get dictionary showing number functions that all arrays will use */ @@ -289,36 +209,18 @@ PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, NPY_NO_EXPORT PyObject * PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op) { + /* + * I suspect that the next few lines are buggy and cause NotImplemented to + * be returned at weird times... but if we raise an error here, then + * *everything* breaks. (Like, 'arange(10) + 1' and just + * 'repr(arange(10))' both blow up with an error here.) Not sure what's + * going on with that, but I'll leave it alone for now. - njs, 2015-06-21 + */ if (op == NULL) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - if (!PyArray_Check(m2) && !has_ufunc_attr(m2)) { - /* - * Catch priority inversion and punt, but only if it's guaranteed - * that we were called through m1 and the other guy is not an array - * at all. Note that some arrays need to pass through here even - * with priorities inverted, for example: float(17) * np.matrix(...) - * - * See also: - * - https://github.com/numpy/numpy/issues/3502 - * - https://github.com/numpy/numpy/issues/3503 - * - * NB: there's another copy of this code in - * numpy.ma.core.MaskedArray._delegate_binop - * which should possibly be updated when this is. - */ - double m1_prio = PyArray_GetPriority((PyObject *)m1, - NPY_SCALAR_PRIORITY); - double m2_prio = PyArray_GetPriority((PyObject *)m2, - NPY_SCALAR_PRIORITY); - if (m1_prio < m2_prio) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - } - return PyObject_CallFunctionObjArgs(op, m1, m2, NULL); } @@ -381,33 +283,21 @@ array_inplace_right_shift(PyArrayObject *m1, PyObject *m2); static PyObject * array_add(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__add__", "__radd__", 0, nb_add); - if (try_binary_elide(m1, m2, &array_inplace_add, &res, 1)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_add, array_add); return PyArray_GenericBinaryFunction(m1, m2, n_ops.add); } static PyObject * array_subtract(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__sub__", "__rsub__", 0, nb_subtract); - if (try_binary_elide(m1, m2, &array_inplace_subtract, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_subtract, array_subtract); return PyArray_GenericBinaryFunction(m1, m2, n_ops.subtract); } static PyObject * array_multiply(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mul__", "__rmul__", 0, nb_multiply); - if (try_binary_elide(m1, m2, &array_inplace_multiply, &res, 1)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, array_multiply); return PyArray_GenericBinaryFunction(m1, m2, n_ops.multiply); } @@ -415,11 +305,7 @@ array_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__div__", "__rdiv__", 0, nb_divide); - if (try_binary_elide(m1, m2, &array_inplace_divide, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_divide, array_divide); return PyArray_GenericBinaryFunction(m1, m2, n_ops.divide); } #endif @@ -427,7 +313,7 @@ array_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__mod__", "__rmod__", 0, nb_remainder); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_remainder, array_remainder); return PyArray_GenericBinaryFunction(m1, m2, n_ops.remainder); } @@ -443,8 +329,7 @@ array_matrix_multiply(PyArrayObject *m1, PyObject *m2) if (matmul == NULL) { return NULL; } - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__matmul__", "__rmatmul__", - 0, nb_matrix_multiply); + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); return PyArray_GenericBinaryFunction(m1, m2, matmul); } @@ -615,12 +500,14 @@ static PyObject * array_power(PyArrayObject *a1, PyObject *o2, PyObject *modulo) { PyObject *value; + if (modulo != Py_None) { /* modular exponentiation is not implemented (gh-8804) */ Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__pow__", "__rpow__", 0, nb_power); + + BINOP_GIVE_UP_IF_NEEDED(a1, o2, nb_power, array_power); value = fast_scalar_power(a1, o2, 0); if (!value) { value = PyArray_GenericBinaryFunction(a1, o2, n_ops.power); @@ -659,76 +546,53 @@ array_invert(PyArrayObject *m1) static PyObject * array_left_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__lshift__", "__rlshift__", 0, nb_lshift); - if (try_binary_elide(m1, m2, &array_inplace_left_shift, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_lshift, array_left_shift); return PyArray_GenericBinaryFunction(m1, m2, n_ops.left_shift); } static PyObject * array_right_shift(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__rshift__", "__rrshift__", 0, nb_rshift); - if (try_binary_elide(m1, m2, &array_inplace_right_shift, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_rshift, array_right_shift); return PyArray_GenericBinaryFunction(m1, m2, n_ops.right_shift); } static PyObject * array_bitwise_and(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__and__", "__rand__", 0, nb_and); - if (try_binary_elide(m1, m2, &array_inplace_bitwise_and, &res, 1)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_and, array_bitwise_and); return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_and); } static PyObject * array_bitwise_or(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__or__", "__ror__", 0, nb_or); - if (try_binary_elide(m1, m2, &array_inplace_bitwise_or, &res, 1)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_or, array_bitwise_or); return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_or); } static PyObject * array_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__xor__", "__rxor__", 0, nb_xor); - if (try_binary_elide(m1, m2, &array_inplace_bitwise_xor, &res, 1)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_xor, array_bitwise_xor); return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_xor); } static PyObject * array_inplace_add(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iadd__", "__radd__", 1, nb_inplace_add); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.add); } static PyObject * array_inplace_subtract(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__isub__", "__rsub__", 1, nb_inplace_subtract); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.subtract); } static PyObject * array_inplace_multiply(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imul__", "__rmul__", 1, nb_inplace_multiply); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.multiply); } @@ -736,7 +600,6 @@ array_inplace_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__idiv__", "__rdiv__", 1, nb_inplace_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.divide); } #endif @@ -744,7 +607,6 @@ array_inplace_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_remainder(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__imod__", "__rmod__", 1, nb_inplace_remainder); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.remainder); } @@ -753,7 +615,6 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo { /* modulo is ignored! */ PyObject *value; - GIVE_UP_IF_HAS_RIGHT_BINOP(a1, o2, "__ipow__", "__rpow__", 1, nb_inplace_power); value = fast_scalar_power(a1, o2, 1); if (!value) { value = PyArray_GenericInplaceBinaryFunction(a1, o2, n_ops.power); @@ -764,66 +625,50 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo static PyObject * array_inplace_left_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ilshift__", "__rlshift__", 1, nb_inplace_lshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.left_shift); } static PyObject * array_inplace_right_shift(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__irshift__", "__rrshift__", 1, nb_inplace_rshift); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.right_shift); } static PyObject * array_inplace_bitwise_and(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__iand__", "__rand__", 1, nb_inplace_and); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_and); } static PyObject * array_inplace_bitwise_or(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ior__", "__ror__", 1, nb_inplace_or); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_or); } static PyObject * array_inplace_bitwise_xor(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ixor__", "__rxor__", 1, nb_inplace_xor); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.bitwise_xor); } static PyObject * array_floor_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__floordiv__", "__rfloordiv__", 0, nb_floor_divide); - if (try_binary_elide(m1, m2, &array_inplace_floor_divide, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_floor_divide, array_floor_divide); return PyArray_GenericBinaryFunction(m1, m2, n_ops.floor_divide); } static PyObject * array_true_divide(PyArrayObject *m1, PyObject *m2) { - PyObject * res; - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__truediv__", "__rtruediv__", 0, nb_true_divide); - if (PyArray_CheckExact(m1) && - (PyArray_ISFLOAT(m1) || PyArray_ISCOMPLEX(m1)) && - try_binary_elide(m1, m2, &array_inplace_true_divide, &res, 0)) { - return res; - } + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_true_divide, array_true_divide); return PyArray_GenericBinaryFunction(m1, m2, n_ops.true_divide); } static PyObject * array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__ifloordiv__", "__rfloordiv__", 1, nb_inplace_floor_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.floor_divide); } @@ -831,7 +676,6 @@ array_inplace_floor_divide(PyArrayObject *m1, PyObject *m2) static PyObject * array_inplace_true_divide(PyArrayObject *m1, PyObject *m2) { - GIVE_UP_IF_HAS_RIGHT_BINOP(m1, m2, "__itruediv__", "__rtruediv__", 1, nb_inplace_true_divide); return PyArray_GenericInplaceBinaryFunction(m1, m2, n_ops.true_divide); } @@ -864,7 +708,7 @@ static PyObject * array_divmod(PyArrayObject *op1, PyObject *op2) { PyObject *divp, *modp, *result; - GIVE_UP_IF_HAS_RIGHT_BINOP(op1, op2, "__divmod__", "__rdivmod__", 0, nb_divmod); + BINOP_GIVE_UP_IF_NEEDED(op1, op2, nb_divmod, array_divmod); divp = array_floor_divide(op1, op2); if (divp == NULL) { diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 0c8355e3170d..86f681c10e65 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -65,8 +65,4 @@ NPY_NO_EXPORT PyObject * PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, int rtype, PyArrayObject *out); -NPY_NO_EXPORT int -needs_right_binop_forward(PyObject *self, PyObject *other, - const char *right_name, int is_inplace); - #endif diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 8908e93fc56c..f6bd5f5a7a52 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -27,6 +27,8 @@ #include +#include "binop_override.h" + NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[] = { {PyObject_HEAD_INIT(&PyBoolArrType_Type) 0}, {PyObject_HEAD_INIT(&PyBoolArrType_Type) 1}, @@ -200,6 +202,7 @@ gentype_generic_method(PyObject *self, PyObject *args, PyObject *kwds, static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } @@ -213,6 +216,7 @@ gentype_@name@(PyObject *m1, PyObject *m2) static PyObject * gentype_@name@(PyObject *m1, PyObject *m2) { + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_@name@, gentype_@name@); return PyArray_Type.tp_as_number->nb_@name@(m1, m2); } /**end repeat**/ @@ -257,8 +261,8 @@ gentype_multiply(PyObject *m1, PyObject *m2) } return PySequence_Repeat(m2, repeat); } - /* All normal cases are handled by PyArray's multiply */ + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, gentype_multiply); return PyArray_Type.tp_as_number->nb_multiply(m1, m2); } diff --git a/numpy/core/src/private/binop_override.h b/numpy/core/src/private/binop_override.h new file mode 100644 index 000000000000..bc48fa46ecdf --- /dev/null +++ b/numpy/core/src/private/binop_override.h @@ -0,0 +1,292 @@ +#ifndef __BINOP_OVERRIDE_H +#define __BINOP_OVERRIDE_H + +#include +#include +#include "numpy/arrayobject.h" + +#include "get_attr_string.h" + +/* + * Logic for deciding when binops should return NotImplemented versus when + * they should go ahead and call a ufunc (or similar). + * + * The interaction between binop methods (ndarray.__add__ and friends) and + * ufuncs (which dispatch to __array_ufunc__) is both complicated in its own + * right, and also has complicated historical constraints. + * + * In the very old days, the rules were: + * - If the other argument has a higher __array_priority__, then return + * NotImplemented + * - Otherwise, call the corresponding ufunc. + * - And the ufunc might return NotImplemented based on some complex + * criteria that I won't reproduce here. + * + * Ufuncs no longer return NotImplemented (except in a few marginal situations + * which are being phased out -- see https://github.com/numpy/numpy/pull/5864) + * + * So as of 1.9, the effective rules were: + * - If the other argument has a higher __array_priority__, and is *not* a + * subclass of ndarray, then return NotImplemented. (If it is a subclass, + * the regular Python rules have already given it a chance to run; so if we + * are running, then it means the other argument has already returned + * NotImplemented and is basically asking us to take care of things.) + * - Otherwise call the corresponding ufunc. + * + * We would like to get rid of __array_priority__, and __array_ufunc__ + * provides a large part of a replacement for it. Once __array_ufunc__ is + * widely available, the simplest dispatch rules that might possibly work + * would be: + * - Always call the corresponding ufunc. + * + * But: + * - Doing this immediately would break backwards compatibility -- there's a + * lot of code using __array_priority__ out there. + * - It's not at all clear whether __array_ufunc__ actually is sufficient for + * all use cases. (See https://github.com/numpy/numpy/issues/5844 for lots + * of discussion of this, and in particular + * https://github.com/numpy/numpy/issues/5844#issuecomment-112014014 + * for a summary of some conclusions.) + * + * So for 1.10, we are going to try the following rules. a.__add__(b) will + * be implemented as follows: + * - If b does not define __array_ufunc__, apply the legacy rule: + * - If not isinstance(b, a.__class__), and b.__array_priority__ is higher + * than a.__array_priority__, return NotImplemented + * - Otherwise, fall through. + * - If b->ob_type["__module__"].startswith("scipy.sparse."), then return + * NotImplemented. (Rationale: scipy.sparse defines __mul__ and np.multiply + * to do two totally different things. We want to grandfather this behavior + * in, but we don't want to support it in the long run, as per PEP + * 465. Additionally, several versions of scipy.sparse were released with + * __array_ufunc__ implementations that don't match the final interface, and + * we don't want dense + sparse to suddenly start erroring out because + * dense.__add__ dispatched to a broken sparse.__array_ufunc__.) + * - Otherwise, call the corresponding ufunc. + * + * For reversed operations like b.__radd__(a), and for in-place operations + * like a.__iadd__(b), we: + * - Call the corresponding ufunc + * + * Rationale for __radd__: This is because by the time the reversed operation + * is called, there are only two possibilities: The first possibility is that + * the current class is a strict subclass of the other class. In practice, the + * only way this will happen is if b is a strict subclass of a, and a is + * ndarray or a subclass of ndarray, and neither a nor b has actually + * overridden this method. In this case, Python will never call a.__add__ + * (because it's identical to b.__radd__), so we have no-one to defer to; + * there's no reason to return NotImplemented. The second possibility is that + * a.__add__ has already been called and returned NotImplemented. Again, in + * this case there is no point in returning NotImplemented. + * + * Rationale for __iadd__: In-place operations do not take all the trouble + * above, because if __iadd__ returns NotImplemented then Python will silently + * convert the operation into an out-of-place operation, i.e. 'a += b' will + * silently become 'a = a + b'. We don't want to allow this for arrays, + * because it will create unexpected memory allocations, break views, + * etc. + * + * In the future we might change these rules further. For example, we plan to + * eventually deprecate __array_priority__ in cases where __array_ufunc__ is + * not present, and we might decide that we need somewhat more flexible + * dispatch rules where the ndarray binops sometimes return NotImplemented + * rather than always dispatching to ufuncs. + * + * Note that these rules are also implemented by ABCArray, so any changes here + * should also be reflected there. + */ + +static int +binop_override_has_ufunc_attr(PyObject *obj) { + PyObject *attr; + int result; + + /* attribute check is expensive for scalar operations, avoid if possible */ + if (PyArray_CheckExact(obj) || PyArray_CheckAnyScalarExact(obj) || + _is_basic_python_type(obj)) { + return 0; + } + + attr = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); + if (attr == NULL) { + return 0; + } + else { + /* + * Pretend that non-callable __array_ufunc__ isn't there. This is an + * escape hatch in case we want to assign some special meaning to + * something like __array_ufunc__ = None, later on. (And can be + * deleted if we decide we don't want to do that.) See these two + * comments: + * https://github.com/numpy/numpy/issues/5844#issuecomment-105081603 + * https://github.com/numpy/numpy/issues/5844#issuecomment-105170926 + */ + result = PyCallable_Check(attr); + Py_DECREF(attr); + return result; + } +} + +static int +binop_override_is_scipy_sparse(PyObject *obj) { + PyObject *module_name = NULL; + PyObject *bytes = NULL; + int result = 0; + char *contents; + + module_name = PyArray_GetAttrString_SuppressException( + (PyObject*) Py_TYPE(obj), + "__module__"); + if (module_name == NULL) { + goto done; + } + if (PyBytes_CheckExact(module_name)) { + contents = PyBytes_AS_STRING(module_name); + } +#if PY_VERSION_HEX >= 0x03020000 + else if (PyUnicode_CheckExact(module_name)) { +#if (PY_VERSION_HEX >= 0x03020000) && (PY_VERSION_HEX < 0x03030000) + /* Python 3.2: unicode, but old API */ + bytes = PyUnicode_AsLatin1String(module_name); + if (bytes == NULL) { + PyErr_Clear(); + goto done; + } + contents = PyString_AS_STRING(bytes); +#endif /* cpython == 3.2.x */ +#if PY_VERSION_HEX >= 0x03030000 + /* Python 3.3+: new unicode API */ + if (PyUnicode_READY(module_name) < 0) { + PyErr_Clear(); + goto done; + } + /* + * We assume that scipy.sparse modules will always have ascii names + */ + if (PyUnicode_KIND(module_name) != PyUnicode_1BYTE_KIND) { + goto done; + } + contents = (char*) PyUnicode_1BYTE_DATA(module_name); +#endif /* cpython >= 3.3 */ + } +#endif /* cpython >= 3.2 */ + else { + goto done; + } + if (strncmp("scipy.sparse", contents, 12) == 0) { + result = 1; + } + + done: + Py_XDECREF(module_name); + Py_XDECREF(bytes); + return result; +} + +static int +binop_override_forward_binop_should_defer(PyObject *self, PyObject *other) +{ + /* + * This function assumes that self.__binop__(other) is underway and + * implements the rules described above. Python's C API is funny, and + * makes it tricky to tell whether a given slot is called for __binop__ + * ("forward") or __rbinop__ ("reversed"). You are responsible for + * determining this before calling this function; it only provides the + * logic for forward binop implementations. + */ + + /* + * NB: there's another copy of this code in + * numpy.ma.core.MaskedArray._delegate_binop + * which should possibly be updated when this is. + */ + + if (other == NULL || + self == NULL || + Py_TYPE(self) == Py_TYPE(other) || + PyArray_CheckExact(other) || + PyArray_CheckAnyScalar(other)) { + /* + * Quick cases + */ + return 0; + } + + /* + * Classes with __array_ufunc__ are living in the future, and don't need + * a check for the legacy __array_priority__. And if other.__class__ is a + * subtype of self.__class__, then it's already had a chance to run, so no + * need to defer to it. + */ + if (!binop_override_has_ufunc_attr(other) && + !PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { + double self_prio = PyArray_GetPriority((PyObject *)self, + NPY_SCALAR_PRIORITY); + double other_prio = PyArray_GetPriority((PyObject *)other, + NPY_SCALAR_PRIORITY); + if (self_prio < other_prio) { + return 1; + } + } + if (binop_override_is_scipy_sparse(other)) { + /* Special case grandfathering in scipy.sparse */ + return 1; + } + + return 0; +} + +/* + * A CPython slot like ->tp_as_number->nb_add gets called for *both* forward + * and reversed operations. E.g. + * a + b + * may call + * a->tp_as_number->nb_add(a, b) + * and + * b + a + * may call + * a->tp_as_number->nb_add(b, a) + * and the only way to tell which is which is for a slot implementation 'f' to + * check + * arg1->tp_as_number->nb_add == f + * arg2->tp_as_number->nb_add == f + * If both are true, then CPython will as a special case only call the + * operation once (i.e., it performs both the forward and reversed binops + * simultaneously). This function is mostly intended for figuring out + * whether we are a forward binop that might want to return NotImplemented, + * and in the both-at-once case we never want to return NotImplemented, so in + * that case BINOP_IS_FORWARD returns false. + * + * This is modeled on the checks in CPython's typeobject.c SLOT1BINFULL + * macro. + */ +#define BINOP_IS_FORWARD(m1, m2, SLOT_NAME, test_func) \ + (Py_TYPE(m2)->tp_as_number != NULL && \ + (void*)(Py_TYPE(m2)->tp_as_number->SLOT_NAME) != (void*)(test_func)) + +#define BINOP_GIVE_UP_IF_NEEDED(m1, m2, slot_expr, test_func) \ + do { \ + if (BINOP_IS_FORWARD(m1, m2, slot_expr, test_func) && \ + binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +/* + * For rich comparison operations, it's impossible to distinguish + * between a forward comparison and a reversed/reflected + * comparison. So we assume they are all forward. This only works because the + * logic in binop_override_forward_binop_should_defer is essentially + * asymmetric -- you can never have two duck-array types that each decide to + * defer to the other. + */ +#define RICHCMP_GIVE_UP_IF_NEEDED(m1, m2) \ + do { \ + if (binop_override_forward_binop_should_defer((PyObject*)m1, (PyObject*)m2)) { \ + Py_INCREF(Py_NotImplemented); \ + return Py_NotImplemented; \ + } \ + } while (0) + +#endif diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index 723ee998ae2d..eb75b73753d3 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -23,6 +23,8 @@ #include "numpy/halffloat.h" #include "templ_common.h" +#include "binop_override.h" + /* Basic operations: * * BINARY: @@ -827,6 +829,8 @@ static PyObject * int first; #endif + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_@oper@, @name@_@oper@); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -964,6 +968,8 @@ static PyObject * int first; @type@ out = {@zero@, @zero@}; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1041,6 +1047,8 @@ static PyObject * PyObject *ret; @type@ arg1, arg2, out; + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1102,6 +1110,9 @@ static PyObject * int first; @type@ out = @zero@; + + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); + switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { case 0: break; @@ -1506,6 +1517,8 @@ static PyObject* npy_@name@ arg1, arg2; int out=0; + RICHCMP_GIVE_UP_IF_NEEDED(self, other); + switch(_@name@_convert2_to_ctypes(self, &arg1, other, &arg2)) { case 0: break; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 24e7585e9687..c0d633178cd3 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2883,226 +2883,178 @@ def test_elide_scalar(self): a = np.bool_() assert_(type(~(a & a)) is np.bool_) - def test_ufunc_override_rop_precedence(self): - # Check that __rmul__ and other right-hand operations have - # precedence over __array_ufunc__ - + # ndarray.__rop__ always calls ufunc + # ndarray.__iop__ always calls ufunc + # ndarray.__op__: + # - if other has __array_ufunc__, call ufunc + # - else, if other is not a subclass and has higher array priority, defer + # - else, if other is in scipy.sparse, defer + # - else, call ufunc + def test_ufunc_binop_interaction(self): + # Python method name (without underscores) + # -> (numpy ufunc, has_in_place_version, preferred_dtype) ops = { - '__add__': ('__radd__', np.add, True), - '__sub__': ('__rsub__', np.subtract, True), - '__mul__': ('__rmul__', np.multiply, True), - '__truediv__': ('__rtruediv__', np.true_divide, True), - '__floordiv__': ('__rfloordiv__', np.floor_divide, True), - '__mod__': ('__rmod__', np.remainder, True), - '__divmod__': ('__rdivmod__', None, False), - '__pow__': ('__rpow__', np.power, True), - '__lshift__': ('__rlshift__', np.left_shift, True), - '__rshift__': ('__rrshift__', np.right_shift, True), - '__and__': ('__rand__', np.bitwise_and, True), - '__xor__': ('__rxor__', np.bitwise_xor, True), - '__or__': ('__ror__', np.bitwise_or, True), - '__ge__': ('__le__', np.less_equal, False), - '__gt__': ('__lt__', np.less, False), - '__le__': ('__ge__', np.greater_equal, False), - '__lt__': ('__gt__', np.greater, False), - '__eq__': ('__eq__', np.equal, False), - '__ne__': ('__ne__', np.not_equal, False), + 'add': (np.add, True, float), + 'sub': (np.subtract, True, float), + 'mul': (np.multiply, True, float), + 'truediv': (np.true_divide, True, float), + 'floordiv': (np.floor_divide, True, float), + 'mod': (np.remainder, True, float), + 'divmod': (None, False, float), + 'pow': (np.power, True, int), + 'lshift': (np.left_shift, True, int), + 'rshift': (np.right_shift, True, int), + 'and': (np.bitwise_and, True, int), + 'xor': (np.bitwise_xor, True, int), + 'or': (np.bitwise_or, True, int), + # 'ge': (np.less_equal, False), + # 'gt': (np.less, False), + # 'le': (np.greater_equal, False), + # 'lt': (np.greater, False), + # 'eq': (np.equal, False), + # 'ne': (np.not_equal, False), } - class OtherNdarraySubclass(np.ndarray): + class Coerced(Exception): pass - - class OtherNdarraySubclassWithOverride(np.ndarray): - def __array_ufunc__(self, *a, **kw): - raise AssertionError(("__array_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def check(op_name, ndsubclass): - rop_name, np_op, has_iop = ops[op_name] - - if has_iop: - iop_name = '__i' + op_name[2:] - iop = getattr(operator, iop_name) - - if op_name == "__divmod__": - op = divmod - else: - op = getattr(operator, op_name) - - # Dummy class - def __init__(self, *a, **kw): - pass - - def __array_ufunc__(self, *a, **kw): - raise AssertionError(("__array_ufunc__ %r %r shouldn't have " - "been called!") % (a, kw)) - - def __op__(self, *other): - return "op" - - def __rop__(self, *other): - return "rop" - - if ndsubclass: - bases = (np.ndarray,) + def array_impl(self): + raise Coerced + def op_impl(self, other): + return "forward" + def rop_impl(self, other): + return "reverse" + def iop_impl(self, other): + return "in-place" + + def array_ufunc_impl(self, ufunc, *args, **kwargs): + return ("__array_ufunc__", ufunc, args, kwargs) + + # Create an object with the given base, in the given module, with a + # bunch of placeholder __op__ methods, and optionally a + # __array_ufunc__ and __array_priority__. + def make_obj(base, array_priority, array_ufunc, + alleged_module="__main__"): + class_namespace = {"__array__": array_impl} + if array_priority is not None: + class_namespace["__array_priority__"] = array_priority + for op in ops: + class_namespace["__{0}__".format(op)] = op_impl + class_namespace["__r{0}__".format(op)] = rop_impl + class_namespace["__i{0}__".format(op)] = iop_impl + if array_ufunc is not None: + class_namespace["__array_ufunc__"] = array_ufunc + eval_namespace = {"base": base, + "class_namespace": class_namespace, + "__name__": alleged_module, + } + MyType = eval("type('MyType', (base,), class_namespace)", + eval_namespace) + if issubclass(MyType, np.ndarray): + # Use this range to avoid special case weirdnesses around + # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc. + return np.arange(3, 5).view(MyType) else: - bases = (object,) - - dct = {'__init__': __init__, - '__array_ufunc__': __array_ufunc__, - op_name: __op__} - if op_name != rop_name: - dct[rop_name] = __rop__ - - cls = type("Rop" + rop_name, bases, dct) - - # Check behavior against both bare ndarray objects and a - # ndarray subclasses with and without their own override - obj = cls((1,), buffer=np.ones(1,)) - - arr_objs = [np.array([1]), - np.array([2]).view(OtherNdarraySubclass), - np.array([3]).view(OtherNdarraySubclassWithOverride), - ] - - for arr in arr_objs: - err_msg = "%r %r" % (op_name, arr,) - - # Check that ndarray op gives up if it sees a non-subclass - if not isinstance(obj, arr.__class__): - assert_equal(getattr(arr, op_name)(obj), - NotImplemented, err_msg=err_msg) - - # Check that the Python binops have priority - assert_equal(op(obj, arr), "op", err_msg=err_msg) - if op_name == rop_name: - assert_equal(op(arr, obj), "op", err_msg=err_msg) - else: - assert_equal(op(arr, obj), "rop", err_msg=err_msg) - - # Check that Python binops have priority also for in-place ops - if has_iop: - assert_equal(getattr(arr, iop_name)(obj), - NotImplemented, err_msg=err_msg) - if op_name != "__pow__": - # inplace pow requires the other object to be - # integer-like? - assert_equal(iop(arr, obj), "rop", err_msg=err_msg) - - # Check that ufunc call __array_ufunc__ normally - if np_op is not None: - assert_raises(AssertionError, np_op, arr, obj, - err_msg=err_msg) - assert_raises(AssertionError, np_op, obj, arr, - err_msg=err_msg) - - # Check all binary operations - for op_name in sorted(ops.keys()): - yield check, op_name, True - yield check, op_name, False - - def test_ufunc_override_rop_simple(self): - # Check parts of the binary op overriding behavior in an - # explicit test case that is easier to understand. - class SomeClass(object): - def __array_ufunc__(self, *a, **kw): - return "ufunc" - - def __mul__(self, other): - return 123 - - def __rmul__(self, other): - return 321 - - def __rsub__(self, other): - return "no subs for me" - - def __gt__(self, other): - return "yep" - - def __lt__(self, other): - return "nope" - - class SomeClass2(SomeClass, np.ndarray): - def __array_ufunc__(self, ufunc, method, inputs, **kw): - if ufunc is np.multiply or ufunc is np.bitwise_and: - return "ufunc" - else: - inputs = [np.asarray(self) if i is self else i - for i in inputs] - func = getattr(ufunc, method) - if ('out' in kw) and (kw['out'] is not None): - kw['out'] = np.asarray(kw['out']) - r = func(*inputs, **kw) - x = self.__class__(r.shape, dtype=r.dtype) - x[...] = r - return x - - class SomeClass3(SomeClass2): - def __rsub__(self, other): - return "sub for me" - - arr = np.array([0]) - obj = SomeClass() - obj2 = SomeClass2((1,), dtype=np.int_) - obj2[0] = 9 - obj3 = SomeClass3((1,), dtype=np.int_) - obj3[0] = 4 - - # obj is first, so should get to define outcome. - assert_equal(obj * arr, 123) - # obj is second, but has __array_ufunc__ and defines __rmul__. - assert_equal(arr * obj, 321) - # obj is second, but has __array_ufunc__ and defines __rsub__. - assert_equal(arr - obj, "no subs for me") - # obj is second, but has __array_ufunc__ and defines __lt__. - assert_equal(arr > obj, "nope") - # obj is second, but has __array_ufunc__ and defines __gt__. - assert_equal(arr < obj, "yep") - # Called as a ufunc, obj.__array_ufunc__ is used. - assert_equal(np.multiply(arr, obj), "ufunc") - # obj is second, but has __array_ufunc__ and defines __rmul__. - arr *= obj - assert_equal(arr, 321) - - # obj2 is an ndarray subclass, so CPython takes care of the same rules. - assert_equal(obj2 * arr, 123) - assert_equal(arr * obj2, 321) - assert_equal(arr - obj2, "no subs for me") - assert_equal(arr > obj2, "nope") - assert_equal(arr < obj2, "yep") - # Called as a ufunc, obj2.__array_ufunc__ is called. - assert_equal(np.multiply(arr, obj2), "ufunc") - # Also when the method is not overridden. - assert_equal(arr & obj2, "ufunc") - arr *= obj2 - assert_equal(arr, 321) - - obj2 += 33 - assert_equal(obj2[0], 42) - assert_equal(obj2.sum(), 42) - assert_(isinstance(obj2, SomeClass2)) - - # Obj3 is subclass that defines __rsub__. CPython calls it. - assert_equal(arr - obj3, "sub for me") - assert_equal(obj2 - obj3, "sub for me") - # obj3 is a subclass that defines __rmul__. CPython calls it. - assert_equal(arr * obj3, 321) - # But not here, since obj3.__rmul__ is obj2.__rmul__. - assert_equal(obj2 * obj3, 123) - # And of course, here obj3.__mul__ should be called. - assert_equal(obj3 * obj2, 123) - # obj3 defines __array_ufunc__ but obj3.__radd__ is obj2.__radd__. - # (and both are just ndarray.__radd__); see #4815. - res = obj2 + obj3 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass2)) - # Since obj3 is a subclass, it should have precedence, like CPython - # would give, even though obj2 has __array_ufunc__ and __radd__. - # See gh-4815 and gh-5747. - res = obj3 + obj2 - assert_equal(res, 46) - assert_(isinstance(res, SomeClass3)) + return MyType() + + def check(obj, binop_override_expected, ufunc_override_expected, + check_scalar=True): + for op, (ufunc, has_inplace, dtype) in ops.items(): + check_objs = [np.arange(3, 5, dtype=dtype)] + if check_scalar: + check_objs.append(check_objs[0][0]) + for arr in check_objs: + arr_method = getattr(arr, "__{0}__".format(op)) + def norm(result): + if op == "divmod": + assert_(isinstance(result, tuple)) + return result[0] + else: + return result + if binop_override_expected: + assert_equal(arr_method(obj), NotImplemented) + elif ufunc_override_expected: + assert_equal(norm(arr_method(obj))[0], + "__array_ufunc__") + else: + if (isinstance(obj, np.ndarray) + and not hasattr(obj, "__array_ufunc__")): + # __array__ gets ignored + res = norm(arr_method(obj)) + assert_(res.__class__ is obj.__class__) + else: + assert_raises((TypeError, Coerced), + arr_method, obj) + arr_rmethod = getattr(arr, "__r{0}__".format(op)) + if ufunc_override_expected: + res = norm(arr_rmethod(obj)) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + else: + if (isinstance(obj, np.ndarray) and + not hasattr(obj, "__array_ufunc__")): + # __array__ gets ignored + res = norm(arr_rmethod(obj)) + assert_(res.__class__ is obj.__class__) + else: + # __array_ufunc__ = "asdf" creates a TypeError + assert_raises((TypeError, Coerced), + arr_rmethod, obj) + # array scalars don't have in-place operators + if has_inplace and isinstance(arr, np.ndarray): + arr_imethod = getattr(arr, "__i{0}__".format(op)) + if ufunc_override_expected: + res = arr_imethod(obj) + assert_equal(res[0], "__array_ufunc__") + if ufunc is not None: + assert_equal(res[1], ufunc) + assert_(res[-1]["out"] is arr) + else: + if (isinstance(obj, np.ndarray) and + not hasattr(obj, "__array_ufunc__")): + # __array__ gets ignored + assert_(arr_imethod(obj) is arr) + else: + assert_raises((TypeError, Coerced), + arr_imethod, obj) + + import operator + op_fn = getattr(operator, op, None) + if op_fn is None: + op_fn = getattr(operator, op + "_", None) + if op_fn is None: + op_fn = getattr(builtins, op) + assert_equal(op_fn(obj, arr), "forward") + if not isinstance(obj, np.ndarray): + if binop_override_expected: + assert_equal(op_fn(arr, obj), "reverse") + elif ufunc_override_expected: + assert_equal(norm(op_fn(arr, obj))[0], + "__array_ufunc__") + if ufunc_override_expected and ufunc is not None: + assert_equal(norm(ufunc(obj, arr))[0], + "__array_ufunc__") + + # No array priority, no numpy ufunc -> nothing called + check(make_obj(object, None, None), False, False) + # Negative array priority, no numpy ufunc -> nothing called + # (has to be very negative, because scalar priority is -1000000.0) + check(make_obj(object, -2**30, None), False, False) + # Positive array priority, no numpy ufunc -> binops only + check(make_obj(object, 1, None), True, False) + # ndarray ignores array priority for ndarray subclasses + check(make_obj(np.ndarray, 1, None), False, False, check_scalar=False) + # Positive array priority and numpy ufunc -> numpy ufunc only + check(make_obj(object, 1, array_ufunc_impl), False, True) + check(make_obj(np.ndarray, 1, array_ufunc_impl), False, True) + # But a non-callable array_ufunc -> like no array_ufunc at all + check(make_obj(object, 1, "asdf"), True, False) + check(make_obj(np.ndarray, 1, "asdf"), False, False, check_scalar=False) + # Objects in scipy.sparse are special: for them we can do both binops + # and ufunc: + check(make_obj(object, 1, array_ufunc_impl, + alleged_module="scipy.sparse"), + True, True) def test_ufunc_override_normalize_signature(self): # gh-5674 @@ -3120,7 +3072,7 @@ def __array_ufunc__(self, ufunc, method, inputs, **kw): assert_('sig' not in kw and 'signature' in kw) assert_equal(kw['signature'], 'ii->i') - def test_numpy_ufunc_index(self): + def test_array_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) class CheckIndex(object): @@ -5303,7 +5255,7 @@ def test_matrix_matrix_values(self): res = self.matmul(m12, m21) assert_equal(res, tgt12_21) - def test_numpy_ufunc_override(self): + def test_array_ufunc_override(self): class A(np.ndarray): def __new__(cls, *args, **kwargs): diff --git a/numpy/ma/core.py b/numpy/ma/core.py index c935edc78760..51b1720821ab 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -3933,12 +3933,15 @@ def __repr__(self): def _delegate_binop(self, other): # This emulates the logic in - # multiarray/number.c:PyArray_GenericBinaryFunction - if (not isinstance(other, np.ndarray) - and not hasattr(other, "__array_ufunc__")): + # private/binop_override.h:forward_binop_should_defer + if isinstance(other, type(self)): + return False + if not hasattr(other, "__array_ufunc__"): other_priority = getattr(other, "__array_priority__", -1000000) if self.__array_priority__ < other_priority: return True + if other.__class__.__module__.startswith("scipy.sparse"): + return True return False def _comparison(self, other, compare): From e4b51639bc4715a8d27e5449e498d6a2fe510f39 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sun, 12 Mar 2017 14:56:44 -0400 Subject: [PATCH 07/43] MAINT: allow __array_ufunc__ = None to force binops to defer. In previous versions, one could force ndarray binops to defer by setting a high __array_priority__. With __array_ufunc__ this gets ignored, and this commit ensures it is still possible to avoid using the standard python language feature that setting something to None means it is not implemented. In consequence, inside a ufunc, if __array_ufunc__ is None, it will be treated as if it had returned NotImplemented (leading to a TypeError if no other object had a functioning __array_ufunc__ override). --- numpy/core/src/private/binop_override.h | 160 +++++------------------- numpy/core/src/private/ufunc_override.h | 14 ++- numpy/core/tests/test_multiarray.py | 111 +++++++++------- numpy/ma/core.py | 13 +- 4 files changed, 117 insertions(+), 181 deletions(-) diff --git a/numpy/core/src/private/binop_override.h b/numpy/core/src/private/binop_override.h index bc48fa46ecdf..8b4458777d5a 100644 --- a/numpy/core/src/private/binop_override.h +++ b/numpy/core/src/private/binop_override.h @@ -46,22 +46,16 @@ * all use cases. (See https://github.com/numpy/numpy/issues/5844 for lots * of discussion of this, and in particular * https://github.com/numpy/numpy/issues/5844#issuecomment-112014014 - * for a summary of some conclusions.) + * for a summary of some conclusions.) Also, python 3.6 defines a standard + * where setting a special-method name to None is a signal that that method + * cannot be used. * - * So for 1.10, we are going to try the following rules. a.__add__(b) will + * So for 1.13, we are going to try the following rules. a.__add__(b) will * be implemented as follows: * - If b does not define __array_ufunc__, apply the legacy rule: * - If not isinstance(b, a.__class__), and b.__array_priority__ is higher * than a.__array_priority__, return NotImplemented - * - Otherwise, fall through. - * - If b->ob_type["__module__"].startswith("scipy.sparse."), then return - * NotImplemented. (Rationale: scipy.sparse defines __mul__ and np.multiply - * to do two totally different things. We want to grandfather this behavior - * in, but we don't want to support it in the long run, as per PEP - * 465. Additionally, several versions of scipy.sparse were released with - * __array_ufunc__ implementations that don't match the final interface, and - * we don't want dense + sparse to suddenly start erroring out because - * dense.__add__ dispatched to a broken sparse.__array_ufunc__.) + * - If b does define __array_ufunc__ but it is None, return NotImplemented * - Otherwise, call the corresponding ufunc. * * For reversed operations like b.__radd__(a), and for in-place operations @@ -76,7 +70,7 @@ * overridden this method. In this case, Python will never call a.__add__ * (because it's identical to b.__radd__), so we have no-one to defer to; * there's no reason to return NotImplemented. The second possibility is that - * a.__add__ has already been called and returned NotImplemented. Again, in + * b.__add__ has already been called and returned NotImplemented. Again, in * this case there is no point in returning NotImplemented. * * Rationale for __iadd__: In-place operations do not take all the trouble @@ -88,101 +82,9 @@ * * In the future we might change these rules further. For example, we plan to * eventually deprecate __array_priority__ in cases where __array_ufunc__ is - * not present, and we might decide that we need somewhat more flexible - * dispatch rules where the ndarray binops sometimes return NotImplemented - * rather than always dispatching to ufuncs. - * - * Note that these rules are also implemented by ABCArray, so any changes here - * should also be reflected there. + * not present. */ -static int -binop_override_has_ufunc_attr(PyObject *obj) { - PyObject *attr; - int result; - - /* attribute check is expensive for scalar operations, avoid if possible */ - if (PyArray_CheckExact(obj) || PyArray_CheckAnyScalarExact(obj) || - _is_basic_python_type(obj)) { - return 0; - } - - attr = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); - if (attr == NULL) { - return 0; - } - else { - /* - * Pretend that non-callable __array_ufunc__ isn't there. This is an - * escape hatch in case we want to assign some special meaning to - * something like __array_ufunc__ = None, later on. (And can be - * deleted if we decide we don't want to do that.) See these two - * comments: - * https://github.com/numpy/numpy/issues/5844#issuecomment-105081603 - * https://github.com/numpy/numpy/issues/5844#issuecomment-105170926 - */ - result = PyCallable_Check(attr); - Py_DECREF(attr); - return result; - } -} - -static int -binop_override_is_scipy_sparse(PyObject *obj) { - PyObject *module_name = NULL; - PyObject *bytes = NULL; - int result = 0; - char *contents; - - module_name = PyArray_GetAttrString_SuppressException( - (PyObject*) Py_TYPE(obj), - "__module__"); - if (module_name == NULL) { - goto done; - } - if (PyBytes_CheckExact(module_name)) { - contents = PyBytes_AS_STRING(module_name); - } -#if PY_VERSION_HEX >= 0x03020000 - else if (PyUnicode_CheckExact(module_name)) { -#if (PY_VERSION_HEX >= 0x03020000) && (PY_VERSION_HEX < 0x03030000) - /* Python 3.2: unicode, but old API */ - bytes = PyUnicode_AsLatin1String(module_name); - if (bytes == NULL) { - PyErr_Clear(); - goto done; - } - contents = PyString_AS_STRING(bytes); -#endif /* cpython == 3.2.x */ -#if PY_VERSION_HEX >= 0x03030000 - /* Python 3.3+: new unicode API */ - if (PyUnicode_READY(module_name) < 0) { - PyErr_Clear(); - goto done; - } - /* - * We assume that scipy.sparse modules will always have ascii names - */ - if (PyUnicode_KIND(module_name) != PyUnicode_1BYTE_KIND) { - goto done; - } - contents = (char*) PyUnicode_1BYTE_DATA(module_name); -#endif /* cpython >= 3.3 */ - } -#endif /* cpython >= 3.2 */ - else { - goto done; - } - if (strncmp("scipy.sparse", contents, 12) == 0) { - result = 1; - } - - done: - Py_XDECREF(module_name); - Py_XDECREF(bytes); - return result; -} - static int binop_override_forward_binop_should_defer(PyObject *self, PyObject *other) { @@ -201,39 +103,41 @@ binop_override_forward_binop_should_defer(PyObject *self, PyObject *other) * which should possibly be updated when this is. */ + PyObject *attr; + double self_prio, other_prio; + int defer; + /* + * attribute check is expensive for scalar operations, avoid if possible + */ if (other == NULL || self == NULL || Py_TYPE(self) == Py_TYPE(other) || PyArray_CheckExact(other) || - PyArray_CheckAnyScalar(other)) { - /* - * Quick cases - */ + PyArray_CheckAnyScalarExact(other) || + _is_basic_python_type(other)) { return 0; } - /* - * Classes with __array_ufunc__ are living in the future, and don't need - * a check for the legacy __array_priority__. And if other.__class__ is a - * subtype of self.__class__, then it's already had a chance to run, so no - * need to defer to it. + * Classes with __array_ufunc__ are living in the future, and only need to + * check whether __array_ufunc__ equals None. */ - if (!binop_override_has_ufunc_attr(other) && - !PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { - double self_prio = PyArray_GetPriority((PyObject *)self, - NPY_SCALAR_PRIORITY); - double other_prio = PyArray_GetPriority((PyObject *)other, - NPY_SCALAR_PRIORITY); - if (self_prio < other_prio) { - return 1; - } + attr = PyArray_GetAttrString_SuppressException(other, "__array_ufunc__"); + if (attr) { + defer = (attr == Py_None); + Py_DECREF(attr); + return defer; } - if (binop_override_is_scipy_sparse(other)) { - /* Special case grandfathering in scipy.sparse */ - return 1; + /* + * Otherwise, we need to check for the legacy __array_priority__. But if + * other.__class__ is a subtype of self.__class__, then it's already had + * a chance to run, so no need to defer to it. + */ + if(PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { + return 0; } - - return 0; + self_prio = PyArray_GetPriority((PyObject *)self, NPY_SCALAR_PRIORITY); + other_prio = PyArray_GetPriority((PyObject *)other, NPY_SCALAR_PRIORITY); + return self_prio < other_prio; } /* diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 357debf4f23a..15e5893886b9 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -165,9 +165,9 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, * Check a set of args for the `__array_ufunc__` method. If more than one of * the input arguments implements `__array_ufunc__`, they are tried in the * order: subclasses before superclasses, otherwise left to right. The first - * routine returning something other than `NotImplemented` determines the - * result. If all of the `__array_ufunc__` operations returns `NotImplemented`, - * a `TypeError` is raised. + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. * * Returns 0 on success and 1 on exception. On success, *result contains the * result of the operation, if any. If *result is NULL, there is no override. @@ -356,13 +356,19 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - /* Call the override */ + /* Access the override */ array_ufunc = PyObject_GetAttrString(override_obj, "__array_ufunc__"); if (array_ufunc == NULL) { goto fail; } + /* If None, try next one (i.e., as if it returned NotImplemented) */ + if (array_ufunc == Py_None) { + Py_DECREF(array_ufunc); + continue; + } + override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); if (override_args == NULL) { Py_DECREF(array_ufunc); diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c0d633178cd3..3e6614661f6c 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2407,22 +2407,28 @@ def test_dot(self): assert_equal(c, np.dot(a, b)) def test_dot_override(self): - class A(object): + class B(object): def __array_ufunc__(self, ufunc, method, inputs, **kwargs): - return "A" + return "B" - class B(object): + class C(object): def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return NotImplemented - a = A() + class D(object): + __array_ufunc__ = None + + a = np.array([[1]]) b = B() - c = np.array([[1]]) + c = C() + d = D() - assert_equal(np.dot(a, b), "A") - assert_equal(c.dot(a), "A") - assert_raises(TypeError, np.dot, b, c) - assert_raises(TypeError, c.dot, b) + assert_equal(np.dot(a, b), "B") + assert_equal(a.dot(b), "B") + assert_raises(TypeError, np.dot, c, a) + assert_raises(TypeError, a.dot, c) + assert_raises(TypeError, np.dot, d, a) + assert_raises(TypeError, a.dot, d) def test_dot_type_mismatch(self): c = 1. @@ -2885,10 +2891,9 @@ def test_elide_scalar(self): # ndarray.__rop__ always calls ufunc # ndarray.__iop__ always calls ufunc - # ndarray.__op__: - # - if other has __array_ufunc__, call ufunc - # - else, if other is not a subclass and has higher array priority, defer - # - else, if other is in scipy.sparse, defer + # ndarray.__op__, __rop__: + # - defer if other has __array_ufunc__ and it is None + # or other is not a subclass and has higher array priority # - else, call ufunc def test_ufunc_binop_interaction(self): # Python method name (without underscores) @@ -2917,12 +2922,16 @@ def test_ufunc_binop_interaction(self): class Coerced(Exception): pass + def array_impl(self): raise Coerced + def op_impl(self, other): return "forward" + def rop_impl(self, other): return "reverse" + def iop_impl(self, other): return "in-place" @@ -2932,16 +2941,16 @@ def array_ufunc_impl(self, ufunc, *args, **kwargs): # Create an object with the given base, in the given module, with a # bunch of placeholder __op__ methods, and optionally a # __array_ufunc__ and __array_priority__. - def make_obj(base, array_priority, array_ufunc, + def make_obj(base, array_priority=False, array_ufunc=False, alleged_module="__main__"): class_namespace = {"__array__": array_impl} - if array_priority is not None: + if array_priority is not False: class_namespace["__array_priority__"] = array_priority for op in ops: class_namespace["__{0}__".format(op)] = op_impl class_namespace["__r{0}__".format(op)] = rop_impl class_namespace["__i{0}__".format(op)] = iop_impl - if array_ufunc is not None: + if array_ufunc is not False: class_namespace["__array_ufunc__"] = array_ufunc eval_namespace = {"base": base, "class_namespace": class_namespace, @@ -2964,26 +2973,29 @@ def check(obj, binop_override_expected, ufunc_override_expected, check_objs.append(check_objs[0][0]) for arr in check_objs: arr_method = getattr(arr, "__{0}__".format(op)) + def norm(result): if op == "divmod": assert_(isinstance(result, tuple)) return result[0] else: return result + if binop_override_expected: assert_equal(arr_method(obj), NotImplemented) elif ufunc_override_expected: assert_equal(norm(arr_method(obj))[0], "__array_ufunc__") else: - if (isinstance(obj, np.ndarray) - and not hasattr(obj, "__array_ufunc__")): + if (isinstance(obj, np.ndarray) and + not hasattr(obj, "__array_ufunc__")): # __array__ gets ignored res = norm(arr_method(obj)) assert_(res.__class__ is obj.__class__) else: assert_raises((TypeError, Coerced), arr_method, obj) + arr_rmethod = getattr(arr, "__r{0}__".format(op)) if ufunc_override_expected: res = norm(arr_rmethod(obj)) @@ -2992,7 +3004,7 @@ def norm(result): assert_equal(res[1], ufunc) else: if (isinstance(obj, np.ndarray) and - not hasattr(obj, "__array_ufunc__")): + not hasattr(obj, "__array_ufunc__")): # __array__ gets ignored res = norm(arr_rmethod(obj)) assert_(res.__class__ is obj.__class__) @@ -3000,6 +3012,7 @@ def norm(result): # __array_ufunc__ = "asdf" creates a TypeError assert_raises((TypeError, Coerced), arr_rmethod, obj) + # array scalars don't have in-place operators if has_inplace and isinstance(arr, np.ndarray): arr_imethod = getattr(arr, "__i{0}__".format(op)) @@ -3011,14 +3024,13 @@ def norm(result): assert_(res[-1]["out"] is arr) else: if (isinstance(obj, np.ndarray) and - not hasattr(obj, "__array_ufunc__")): + not hasattr(obj, "__array_ufunc__")): # __array__ gets ignored assert_(arr_imethod(obj) is arr) else: assert_raises((TypeError, Coerced), arr_imethod, obj) - import operator op_fn = getattr(operator, op, None) if op_fn is None: op_fn = getattr(operator, op + "_", None) @@ -3036,25 +3048,24 @@ def norm(result): "__array_ufunc__") # No array priority, no numpy ufunc -> nothing called - check(make_obj(object, None, None), False, False) + check(make_obj(object), False, False) # Negative array priority, no numpy ufunc -> nothing called # (has to be very negative, because scalar priority is -1000000.0) - check(make_obj(object, -2**30, None), False, False) + check(make_obj(object, array_priority=-2**30), False, False) # Positive array priority, no numpy ufunc -> binops only - check(make_obj(object, 1, None), True, False) + check(make_obj(object, array_priority=1), True, False) # ndarray ignores array priority for ndarray subclasses - check(make_obj(np.ndarray, 1, None), False, False, check_scalar=False) + check(make_obj(np.ndarray, array_priority=1), False, False, + check_scalar=False) # Positive array priority and numpy ufunc -> numpy ufunc only - check(make_obj(object, 1, array_ufunc_impl), False, True) - check(make_obj(np.ndarray, 1, array_ufunc_impl), False, True) - # But a non-callable array_ufunc -> like no array_ufunc at all - check(make_obj(object, 1, "asdf"), True, False) - check(make_obj(np.ndarray, 1, "asdf"), False, False, check_scalar=False) - # Objects in scipy.sparse are special: for them we can do both binops - # and ufunc: - check(make_obj(object, 1, array_ufunc_impl, - alleged_module="scipy.sparse"), - True, True) + check(make_obj(object, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + check(make_obj(np.ndarray, array_priority=1, + array_ufunc=array_ufunc_impl), False, True) + # array_ufunc set to None -> defer binops only + check(make_obj(object, array_ufunc=None), True, False) + check(make_obj(np.ndarray, array_ufunc=None), True, False, + check_scalar=False) def test_ufunc_override_normalize_signature(self): # gh-5674 @@ -5257,26 +5268,40 @@ def test_matrix_matrix_values(self): def test_array_ufunc_override(self): - class A(np.ndarray): + class B(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) def __array_ufunc__(self, ufunc, method, inputs, **kwargs): - return "A" + return "B" - class B(np.ndarray): + class C(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) def __array_ufunc__(self, ufunc, method, inputs, **kwargs): return NotImplemented - a = A([1, 2]) + class D(np.ndarray): + def __new__(cls, *args, **kwargs): + return np.array(*args, **kwargs).view(cls) + + __array_ufunc__ = None + + a = np.ones(2) b = B([1, 2]) - c = np.ones(2) - assert_equal(self.matmul(a, b), "A") - assert_equal(self.matmul(b, a), "A") - assert_raises(TypeError, self.matmul, b, c) + c = C([1, 2]) + d = D([1, 2]) + assert_equal(self.matmul(b, a), "B") + assert_equal(self.matmul(a, b), "B") + assert_equal(self.matmul(b, c), "B") + assert_equal(self.matmul(c, b), "B") + assert_equal(self.matmul(b, d), "B") + assert_equal(self.matmul(d, b), "B") + assert_raises(TypeError, self.matmul, a, c) + assert_raises(TypeError, self.matmul, c, a) + assert_raises(TypeError, self.matmul, d, a) + assert_raises(TypeError, self.matmul, a, d) class TestMatmul(MatmulCommon, TestCase): diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 51b1720821ab..cb0bfdde296a 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -3936,13 +3936,14 @@ def _delegate_binop(self, other): # private/binop_override.h:forward_binop_should_defer if isinstance(other, type(self)): return False - if not hasattr(other, "__array_ufunc__"): + array_ufunc = getattr(other, "__array_ufunc__", False) + if array_ufunc is False: other_priority = getattr(other, "__array_priority__", -1000000) - if self.__array_priority__ < other_priority: - return True - if other.__class__.__module__.startswith("scipy.sparse"): - return True - return False + return self.__array_priority__ < other_priority + else: + # If array_ufunc is not None, it will be called inside the ufunc; + # None explicitly tells us to not call the ufunc, i.e., defer. + return array_ufunc is None def _comparison(self, other, compare): """Compare self with other using operator.eq or operator.ne. From 2e6d8c08b87d7b347231fe617aefdf9fc9f29618 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Wed, 15 Mar 2017 11:59:11 -0400 Subject: [PATCH 08/43] MAINT: Split out C code in ufunc_override.h to .c file. --- numpy/core/setup.py | 9 +- numpy/core/src/private/ufunc_override.c | 406 ++++++++++++++++++++++++ numpy/core/src/private/ufunc_override.h | 406 +----------------------- 3 files changed, 417 insertions(+), 404 deletions(-) create mode 100644 numpy/core/src/private/ufunc_override.c diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 20d4c77922b4..b29a8fee2c27 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -748,6 +748,8 @@ def get_mathlib_info(*args): join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h'), join('src', 'private', 'npy_extint128.h'), join('include', 'numpy', 'arrayobject.h'), join('include', 'numpy', '_neighborhood_iterator_imp.h'), @@ -818,6 +820,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'vdot.c'), join('src', 'private', 'templ_common.h.src'), join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c'), ] blas_info = get_info('blas_opt', 0) @@ -871,7 +874,8 @@ def generate_umath_c(ext, build_dir): join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'scalarmath.c.src'), join('src', 'umath', 'ufunc_type_resolution.c'), - join('src', 'private', 'mem_overlap.c')] + join('src', 'private', 'mem_overlap.c'), + join('src', 'private', 'ufunc_override.c')] umath_deps = [ generate_umath_py, @@ -883,7 +887,8 @@ def generate_umath_c(ext, build_dir): join(codegen_dir, 'generate_ufunc_api.py'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), - join('src', 'private', 'ufunc_override.h')] + npymath_sources + join('src', 'private', 'ufunc_override.h'), + join('src', 'private', 'binop_override.h')] + npymath_sources config.add_extension('umath', sources=umath_src + diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c new file mode 100644 index 000000000000..1c5c8099c653 --- /dev/null +++ b/numpy/core/src/private/ufunc_override.c @@ -0,0 +1,406 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#include "npy_pycompat.h" +#include "numpy/ufuncobject.h" +#include "get_attr_string.h" + +#include "ufunc_override.h" + +static void +normalize___call___args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds, + int nin) +{ + /* ufunc.__call__(*args, **kwds) */ + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); + + /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + if (obj != NULL) { + Py_INCREF(obj); + PyDict_SetItemString(*normal_kwds, "signature", obj); + PyDict_DelItemString(*normal_kwds, "sig"); + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + + /* If we have more args than nin, they must be the output variables.*/ + if (nargs > nin) { + if ((nargs - nin) == 1) { + obj = PyTuple_GET_ITEM(args, nargs - 1); + PyDict_SetItemString(*normal_kwds, "out", obj); + } + else { + obj = PyTuple_GetSlice(args, nin, nargs); + PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); + } + } +} + +static void +normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + + for (i = 0; i < nargs; i++) { + obj = PyTuple_GET_ITEM(args, i); + if (i == 0) { + *normal_args = PyTuple_GetSlice(args, 0, 1); + } + else if (i == 1) { + /* axis */ + PyDict_SetItemString(*normal_kwds, "axis", obj); + } + else if (i == 2) { + /* dtype */ + PyDict_SetItemString(*normal_kwds, "dtype", obj); + } + else if (i == 3) { + /* out */ + PyDict_SetItemString(*normal_kwds, "out", obj); + } + else { + /* keepdims */ + PyDict_SetItemString(*normal_kwds, "keepdims", obj); + } + } + return; +} + +static void +normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.accumulate(a[, axis, dtype, out]) */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + + for (i = 0; i < nargs; i++) { + obj = PyTuple_GET_ITEM(args, i); + if (i == 0) { + *normal_args = PyTuple_GetSlice(args, 0, 1); + } + else if (i == 1) { + /* axis */ + PyDict_SetItemString(*normal_kwds, "axis", obj); + } + else if (i == 2) { + /* dtype */ + PyDict_SetItemString(*normal_kwds, "dtype", obj); + } + else { + /* out */ + PyDict_SetItemString(*normal_kwds, "out", obj); + } + } + return; +} + +static void +normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ + int i; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + + for (i = 0; i < nargs; i++) { + obj = PyTuple_GET_ITEM(args, i); + if (i == 0) { + /* a and indicies */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + } + else if (i == 1) { + /* Handled above, when i == 0. */ + continue; + } + else if (i == 2) { + /* axis */ + PyDict_SetItemString(*normal_kwds, "axis", obj); + } + else if (i == 3) { + /* dtype */ + PyDict_SetItemString(*normal_kwds, "dtype", obj); + } + else { + /* out */ + PyDict_SetItemString(*normal_kwds, "out", obj); + } + } + return; +} + +static void +normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.outer(A, B) + * This has no kwds so we don't need to do any kwd stuff. + */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + return; +} + +static void +normalize_at_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.at(a, indices[, b]) */ + int nargs = PyTuple_GET_SIZE(args); + + *normal_args = PyTuple_GetSlice(args, 0, nargs); + return; +} + +/* + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. + * + * Returns 0 on success and 1 on exception. On success, *result contains the + * result of the operation, if any. If *result is NULL, there is no override. + */ +int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result, + int nin) +{ + int i; + int j; + + int nargs; + int nout_kwd = 0; + int out_kwd_is_tuple = 0; + int noa = 0; /* Number of overriding args.*/ + + PyObject *tmp; + PyObject *obj; + PyObject *out_kwd_obj = NULL; + PyObject *other_obj; + + PyObject *method_name = NULL; + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + + PyObject *with_override[NPY_MAXARGS]; + + /* + * Check inputs + */ + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_ValueError, + "Internal Numpy error: call to PyUFunc_CheckOverride " + "with non-tuple"); + goto fail; + } + nargs = PyTuple_GET_SIZE(args); + if (nargs > NPY_MAXARGS) { + PyErr_SetString(PyExc_ValueError, + "Internal Numpy error: too many arguments in call " + "to PyUFunc_CheckOverride"); + goto fail; + } + + /* be sure to include possible 'out' keyword argument. */ + if ((kwds)&& (PyDict_CheckExact(kwds))) { + out_kwd_obj = PyDict_GetItemString(kwds, "out"); + if (out_kwd_obj != NULL) { + out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); + if (out_kwd_is_tuple) { + nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); + } + else { + nout_kwd = 1; + } + } + } + + for (i = 0; i < nargs + nout_kwd; ++i) { + if (i < nargs) { + obj = PyTuple_GET_ITEM(args, i); + } + else { + if (out_kwd_is_tuple) { + obj = PyTuple_GET_ITEM(out_kwd_obj, i-nargs); + } + else { + obj = out_kwd_obj; + } + } + tmp = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); + if (tmp) { + Py_DECREF(tmp); + with_override[noa] = obj; + ++noa; + } + } + + /* No overrides, bail out.*/ + if (noa == 0) { + *result = NULL; + return 0; + } + + method_name = PyUString_FromString(method); + if (method_name == NULL) { + goto fail; + } + + /* + * Normalize ufunc arguments. + */ + + /* Build new kwds */ + if (kwds && PyDict_CheckExact(kwds)) { + normal_kwds = PyDict_Copy(kwds); + } + else { + normal_kwds = PyDict_New(); + } + if (normal_kwds == NULL) { + goto fail; + } + + /* decide what to do based on the method. */ + /* ufunc.__call__ */ + if (strcmp(method, "__call__") == 0) { + normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); + } + + /* ufunc.reduce */ + else if (strcmp(method, "reduce") == 0) { + normalize_reduce_args(ufunc, args, &normal_args, &normal_kwds); + } + + /* ufunc.accumulate */ + else if (strcmp(method, "accumulate") == 0) { + normalize_accumulate_args(ufunc, args, &normal_args, &normal_kwds); + } + + /* ufunc.reduceat */ + else if (strcmp(method, "reduceat") == 0) { + normalize_reduceat_args(ufunc, args, &normal_args, &normal_kwds); + } + + /* ufunc.outer */ + else if (strcmp(method, "outer") == 0) { + normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + } + + /* ufunc.at */ + else if (strcmp(method, "at") == 0) { + normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + } + + if (normal_args == NULL) { + goto fail; + } + + /* + * Call __array_ufunc__ functions in correct order + */ + while (1) { + PyObject *array_ufunc; + PyObject *override_args; + PyObject *override_obj; + + override_obj = NULL; + *result = NULL; + + /* Choose an overriding argument */ + for (i = 0; i < noa; i++) { + obj = with_override[i]; + if (obj == NULL) { + continue; + } + + /* Get the first instance of an overriding arg.*/ + override_obj = obj; + + /* Check for sub-types to the right of obj. */ + for (j = i + 1; j < noa; j++) { + other_obj = with_override[j]; + if (PyObject_Type(other_obj) != PyObject_Type(obj) && + PyObject_IsInstance(other_obj, + PyObject_Type(override_obj))) { + override_obj = NULL; + break; + } + } + + /* override_obj had no subtypes to the right. */ + if (override_obj) { + with_override[i] = NULL; /* We won't call this one again */ + break; + } + } + + /* Check if there is a method left to call */ + if (!override_obj) { + /* No acceptable override found. */ + PyErr_SetString(PyExc_TypeError, + "__array_ufunc__ not implemented for this type."); + goto fail; + } + + /* Access the override */ + array_ufunc = PyObject_GetAttrString(override_obj, + "__array_ufunc__"); + if (array_ufunc == NULL) { + goto fail; + } + + /* If None, try next one (i.e., as if it returned NotImplemented) */ + if (array_ufunc == Py_None) { + Py_DECREF(array_ufunc); + continue; + } + + override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); + if (override_args == NULL) { + Py_DECREF(array_ufunc); + goto fail; + } + + *result = PyObject_Call(array_ufunc, override_args, normal_kwds); + + Py_DECREF(array_ufunc); + Py_DECREF(override_args); + + if (*result == NULL) { + /* Exception occurred */ + goto fail; + } + else if (*result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(*result); + continue; + } + else { + /* Good result. */ + break; + } + } + + /* Override found, return it. */ + Py_XDECREF(method_name); + Py_XDECREF(normal_args); + Py_XDECREF(normal_kwds); + return 0; + +fail: + Py_XDECREF(method_name); + Py_XDECREF(normal_args); + Py_XDECREF(normal_kwds); + return 1; +} diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 15e5893886b9..4354b3869026 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -1,410 +1,12 @@ #ifndef __UFUNC_OVERRIDE_H #define __UFUNC_OVERRIDE_H -#include -#include "numpy/arrayobject.h" -#include "common.h" -#include -#include "numpy/ufuncobject.h" - -#include "get_attr_string.h" - -static void -normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds, - int nin) -{ - /* ufunc.__call__(*args, **kwds) */ - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); - - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - if (obj != NULL) { - Py_INCREF(obj); - PyDict_SetItemString(*normal_kwds, "signature", obj); - PyDict_DelItemString(*normal_kwds, "sig"); - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - - /* If we have more args than nin, they must be the output variables.*/ - if (nargs > nin) { - if ((nargs - nin) == 1) { - obj = PyTuple_GET_ITEM(args, nargs - 1); - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - obj = PyTuple_GetSlice(args, nin, nargs); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } - } -} - -static void -normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else if (i == 3) { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - else { - /* keepdims */ - PyDict_SetItemString(*normal_kwds, "keepdims", obj); - } - } - return; -} - -static void -normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.accumulate(a[, axis, dtype, out]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); - } - else if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} - -static void -normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ - int i; - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - - for (i = 0; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - /* a and indicies */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - } - else if (i == 1) { - /* Handled above, when i == 0. */ - continue; - } - else if (i == 2) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 3) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - PyDict_SetItemString(*normal_kwds, "out", obj); - } - } - return; -} -static void -normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.outer(A, B) - * This has no kwds so we don't need to do any kwd stuff. - */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - return; -} - -static void -normalize_at_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.at(a, indices[, b]) */ - int nargs = PyTuple_GET_SIZE(args); - - *normal_args = PyTuple_GetSlice(args, 0, nargs); - return; -} +#include "npy_config.h" +#include "numpy/ufuncobject.h" -/* - * Check a set of args for the `__array_ufunc__` method. If more than one of - * the input arguments implements `__array_ufunc__`, they are tried in the - * order: subclasses before superclasses, otherwise left to right. The first - * (non-None) routine returning something other than `NotImplemented` - * determines the result. If all of the `__array_ufunc__` operations return - * `NotImplemented` (or are None), a `TypeError` is raised. - * - * Returns 0 on success and 1 on exception. On success, *result contains the - * result of the operation, if any. If *result is NULL, there is no override. - */ -static int +int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, PyObject **result, - int nin) -{ - int i; - int j; - - int nargs; - int nout_kwd = 0; - int out_kwd_is_tuple = 0; - int noa = 0; /* Number of overriding args.*/ - - PyObject *tmp; - PyObject *obj; - PyObject *out_kwd_obj = NULL; - PyObject *other_obj; - - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ - PyObject *normal_kwds = NULL; - - PyObject *with_override[NPY_MAXARGS]; - - /* - * Check inputs - */ - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: call to PyUFunc_CheckOverride " - "with non-tuple"); - goto fail; - } - nargs = PyTuple_GET_SIZE(args); - if (nargs > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, - "Internal Numpy error: too many arguments in call " - "to PyUFunc_CheckOverride"); - goto fail; - } - - /* be sure to include possible 'out' keyword argument. */ - if ((kwds)&& (PyDict_CheckExact(kwds))) { - out_kwd_obj = PyDict_GetItemString(kwds, "out"); - if (out_kwd_obj != NULL) { - out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); - if (out_kwd_is_tuple) { - nout_kwd = PyTuple_GET_SIZE(out_kwd_obj); - } - else { - nout_kwd = 1; - } - } - } - - for (i = 0; i < nargs + nout_kwd; ++i) { - if (i < nargs) { - obj = PyTuple_GET_ITEM(args, i); - } - else { - if (out_kwd_is_tuple) { - obj = PyTuple_GET_ITEM(out_kwd_obj, i-nargs); - } - else { - obj = out_kwd_obj; - } - } - tmp = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); - if (tmp) { - Py_DECREF(tmp); - with_override[noa] = obj; - ++noa; - } - } - - /* No overrides, bail out.*/ - if (noa == 0) { - *result = NULL; - return 0; - } - - method_name = PyUString_FromString(method); - if (method_name == NULL) { - goto fail; - } - - /* - * Normalize ufunc arguments. - */ - - /* Build new kwds */ - if (kwds && PyDict_CheckExact(kwds)) { - normal_kwds = PyDict_Copy(kwds); - } - else { - normal_kwds = PyDict_New(); - } - if (normal_kwds == NULL) { - goto fail; - } - - /* decide what to do based on the method. */ - /* ufunc.__call__ */ - if (strcmp(method, "__call__") == 0) { - normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); - } - - /* ufunc.reduce */ - else if (strcmp(method, "reduce") == 0) { - normalize_reduce_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.accumulate */ - else if (strcmp(method, "accumulate") == 0) { - normalize_accumulate_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.reduceat */ - else if (strcmp(method, "reduceat") == 0) { - normalize_reduceat_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.outer */ - else if (strcmp(method, "outer") == 0) { - normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); - } - - /* ufunc.at */ - else if (strcmp(method, "at") == 0) { - normalize_at_args(ufunc, args, &normal_args, &normal_kwds); - } - - if (normal_args == NULL) { - goto fail; - } - - /* - * Call __array_ufunc__ functions in correct order - */ - while (1) { - PyObject *array_ufunc; - PyObject *override_args; - PyObject *override_obj; - - override_obj = NULL; - *result = NULL; - - /* Choose an overriding argument */ - for (i = 0; i < noa; i++) { - obj = with_override[i]; - if (obj == NULL) { - continue; - } - - /* Get the first instance of an overriding arg.*/ - override_obj = obj; - - /* Check for sub-types to the right of obj. */ - for (j = i + 1; j < noa; j++) { - other_obj = with_override[j]; - if (PyObject_Type(other_obj) != PyObject_Type(obj) && - PyObject_IsInstance(other_obj, - PyObject_Type(override_obj))) { - override_obj = NULL; - break; - } - } - - /* override_obj had no subtypes to the right. */ - if (override_obj) { - with_override[i] = NULL; /* We won't call this one again */ - break; - } - } - - /* Check if there is a method left to call */ - if (!override_obj) { - /* No acceptable override found. */ - PyErr_SetString(PyExc_TypeError, - "__array_ufunc__ not implemented for this type."); - goto fail; - } - - /* Access the override */ - array_ufunc = PyObject_GetAttrString(override_obj, - "__array_ufunc__"); - if (array_ufunc == NULL) { - goto fail; - } - - /* If None, try next one (i.e., as if it returned NotImplemented) */ - if (array_ufunc == Py_None) { - Py_DECREF(array_ufunc); - continue; - } - - override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); - if (override_args == NULL) { - Py_DECREF(array_ufunc); - goto fail; - } - - *result = PyObject_Call(array_ufunc, override_args, normal_kwds); - - Py_DECREF(array_ufunc); - Py_DECREF(override_args); - - if (*result == NULL) { - /* Exception occurred */ - goto fail; - } - else if (*result == Py_NotImplemented) { - /* Try the next one */ - Py_DECREF(*result); - continue; - } - else { - /* Good result. */ - break; - } - } - - /* Override found, return it. */ - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 0; - -fail: - Py_XDECREF(method_name); - Py_XDECREF(normal_args); - Py_XDECREF(normal_kwds); - return 1; -} + int nin); #endif From d5c5ac179b5e6a9e139df8334857aac0b0b7600e Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Thu, 16 Mar 2017 17:03:58 -0600 Subject: [PATCH 09/43] MAINT: Add NPY_NO_EXPORT modifier to PyUFunc_CheckOverride. --- numpy/core/src/private/ufunc_override.c | 2 +- numpy/core/src/private/ufunc_override.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index 1c5c8099c653..c098458f8e4c 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -169,7 +169,7 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, * Returns 0 on success and 1 on exception. On success, *result contains the * result of the operation, if any. If *result is NULL, there is no override. */ -int +NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, PyObject **result, diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 4354b3869026..ee18e569c34d 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -4,7 +4,7 @@ #include "npy_config.h" #include "numpy/ufuncobject.h" -int +NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, PyObject **result, From 3124e9692aa7cc1aa3baf1f148291bd9ed00d7fd Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Tue, 14 Mar 2017 17:49:14 -0400 Subject: [PATCH 10/43] MAINT: for __array_ufunc__ pass inputs as *args, ensure out is tuple. --- numpy/core/src/private/ufunc_override.c | 141 +++++++++++++++++------- numpy/core/tests/test_multiarray.py | 59 +++++----- numpy/core/tests/test_ufunc.py | 2 +- numpy/core/tests/test_umath.py | 58 +++++----- 4 files changed, 161 insertions(+), 99 deletions(-) diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index c098458f8e4c..655aa27f7f94 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -11,6 +11,8 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, int nin) { /* ufunc.__call__(*args, **kwds) */ + int i; + int not_all_none; int nargs = PyTuple_GET_SIZE(args); PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); @@ -25,11 +27,13 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, /* If we have more args than nin, they must be the output variables.*/ if (nargs > nin) { - if ((nargs - nin) == 1) { - obj = PyTuple_GET_ITEM(args, nargs - 1); - PyDict_SetItemString(*normal_kwds, "out", obj); + for (i=nin; i < nargs; i++) { + not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); + if (not_all_none) { + break; + } } - else { + if (not_all_none) { obj = PyTuple_GetSlice(args, nin, nargs); PyDict_SetItemString(*normal_kwds, "out", obj); Py_DECREF(obj); @@ -41,17 +45,18 @@ static void normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ + /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ int nargs = PyTuple_GET_SIZE(args); int i; PyObject *obj; - for (i = 0; i < nargs; i++) { + *normal_args = PyTuple_GetSlice(args, 0, 1); + for (i = 1; i < nargs; i++) { obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); + if (obj == Py_None) { + continue; } - else if (i == 1) { + if (i == 1) { /* axis */ PyDict_SetItemString(*normal_kwds, "axis", obj); } @@ -61,7 +66,9 @@ normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, } else if (i == 3) { /* out */ + obj = PyTuple_GetSlice(args, 3, 4); PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); } else { /* keepdims */ @@ -75,17 +82,18 @@ static void normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.accumulate(a[, axis, dtype, out]) */ + /* ufunc.accumulate(a[, axis, dtype, out]) */ int nargs = PyTuple_GET_SIZE(args); int i; PyObject *obj; - for (i = 0; i < nargs; i++) { + *normal_args = PyTuple_GetSlice(args, 0, 1); + for (i = 1; i < nargs; i++) { obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - *normal_args = PyTuple_GetSlice(args, 0, 1); + if (obj == Py_None) { + continue; } - else if (i == 1) { + if (i == 1) { /* axis */ PyDict_SetItemString(*normal_kwds, "axis", obj); } @@ -95,7 +103,9 @@ normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, } else { /* out */ + obj = PyTuple_GetSlice(args, 3, 4); PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); } } return; @@ -105,22 +115,20 @@ static void normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ + /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ int i; int nargs = PyTuple_GET_SIZE(args); PyObject *obj; - for (i = 0; i < nargs; i++) { + /* a and indicies */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + + for (i = 2; i < nargs; i++) { obj = PyTuple_GET_ITEM(args, i); - if (i == 0) { - /* a and indicies */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - } - else if (i == 1) { - /* Handled above, when i == 0. */ + if (obj == Py_None) { continue; } - else if (i == 2) { + if (i == 2) { /* axis */ PyDict_SetItemString(*normal_kwds, "axis", obj); } @@ -130,7 +138,9 @@ normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, } else { /* out */ + obj = PyTuple_GetSlice(args, 4, 5); PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); } } return; @@ -140,7 +150,8 @@ static void normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.outer(A, B) + /* + * ufunc.outer(A, B) * This has no kwds so we don't need to do any kwd stuff. */ *normal_args = PyTuple_GetSlice(args, 0, 2); @@ -151,7 +162,7 @@ static void normalize_at_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.at(a, indices[, b]) */ + /* ufunc.at(a, indices[, b]) */ int nargs = PyTuple_GET_SIZE(args); *normal_args = PyTuple_GetSlice(args, 0, nargs); @@ -193,6 +204,8 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *normal_kwds = NULL; PyObject *with_override[NPY_MAXARGS]; + Py_ssize_t len; + PyObject *override_args; /* * Check inputs @@ -212,7 +225,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* be sure to include possible 'out' keyword argument. */ - if ((kwds)&& (PyDict_CheckExact(kwds))) { + if (kwds && PyDict_CheckExact(kwds)) { out_kwd_obj = PyDict_GetItemString(kwds, "out"); if (out_kwd_obj != NULL) { out_kwd_is_tuple = PyTuple_CheckExact(out_kwd_obj); @@ -262,7 +275,41 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Build new kwds */ if (kwds && PyDict_CheckExact(kwds)) { + PyObject *out; + + /* ensure out is always a tuple */ normal_kwds = PyDict_Copy(kwds); + out = PyDict_GetItemString(normal_kwds, "out"); + if (out != NULL) { + if (PyTuple_Check(out)) { + int all_none; + int i; + + for (i = 0; i < PyTuple_GET_SIZE(out); i++) { + all_none = (PyTuple_GET_ITEM(out, i) == Py_None); + if (!all_none) { + break; + } + } + if (all_none) { + PyDict_DelItemString(normal_kwds, "out"); + } + } + else if (out != Py_None) { + /* not already a tuple and not None */ + PyObject *out_tuple = PyTuple_New(1); + + if (out_tuple == NULL) { + goto fail; + } + /* out was borrowed ref; make it permanent */ + Py_INCREF(out); + /* steals reference */ + PyTuple_SET_ITEM(out_tuple, 0, out); + PyDict_SetItemString(normal_kwds, "out", out_tuple); + Py_DECREF(out_tuple); + } + } } else { normal_kwds = PyDict_New(); @@ -272,6 +319,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* decide what to do based on the method. */ + /* ufunc.__call__ */ if (strcmp(method, "__call__") == 0) { normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); @@ -306,12 +354,28 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - /* - * Call __array_ufunc__ functions in correct order - */ + len = PyTuple_GET_SIZE(normal_args); + override_args = PyTuple_New(len + 2); + if (override_args == NULL) { + goto fail; + } + + Py_INCREF(ufunc); + /* PyTuple_SET_ITEM steals reference */ + PyTuple_SET_ITEM(override_args, 0, ufunc); + Py_INCREF(method_name); + PyTuple_SET_ITEM(override_args, 1, method_name); + for (i = 0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(normal_args, i); + + Py_INCREF(item); + PyTuple_SET_ITEM(override_args, i + 2, item); + } + Py_DECREF(normal_args); + + /* Call __array_ufunc__ functions in correct order */ while (1) { PyObject *array_ufunc; - PyObject *override_args; PyObject *override_obj; override_obj = NULL; @@ -340,7 +404,8 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* override_obj had no subtypes to the right. */ if (override_obj) { - with_override[i] = NULL; /* We won't call this one again */ + /* We won't call this one again */ + with_override[i] = NULL; break; } } @@ -360,22 +425,14 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - /* If None, try next one (i.e., as if it returned NotImplemented) */ + /* If None, try next one (i.e., as if it returned NotImplemented) */ if (array_ufunc == Py_None) { Py_DECREF(array_ufunc); continue; } - override_args = Py_BuildValue("OOO", ufunc, method_name, normal_args); - if (override_args == NULL) { - Py_DECREF(array_ufunc); - goto fail; - } - *result = PyObject_Call(array_ufunc, override_args, normal_kwds); - Py_DECREF(array_ufunc); - Py_DECREF(override_args); if (*result == NULL) { /* Exception occurred */ @@ -394,13 +451,13 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Override found, return it. */ Py_XDECREF(method_name); - Py_XDECREF(normal_args); Py_XDECREF(normal_kwds); + Py_DECREF(override_args); return 0; fail: Py_XDECREF(method_name); - Py_XDECREF(normal_args); Py_XDECREF(normal_kwds); + Py_XDECREF(override_args); return 1; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 3e6614661f6c..9766ee47af90 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2408,11 +2408,11 @@ def test_dot(self): def test_dot_override(self): class B(object): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return "B" class C(object): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return NotImplemented class D(object): @@ -2935,8 +2935,8 @@ def rop_impl(self, other): def iop_impl(self, other): return "in-place" - def array_ufunc_impl(self, ufunc, *args, **kwargs): - return ("__array_ufunc__", ufunc, args, kwargs) + def array_ufunc_impl(self, ufunc, method, *args, **kwargs): + return ("__array_ufunc__", ufunc, method, args, kwargs) # Create an object with the given base, in the given module, with a # bunch of placeholder __op__ methods, and optionally a @@ -3021,7 +3021,8 @@ def norm(result): assert_equal(res[0], "__array_ufunc__") if ufunc is not None: assert_equal(res[1], ufunc) - assert_(res[-1]["out"] is arr) + assert_(type(res[-1]["out"]) is tuple) + assert_(res[-1]["out"][0] is arr) else: if (isinstance(obj, np.ndarray) and not hasattr(obj, "__array_ufunc__")): @@ -3070,7 +3071,7 @@ def norm(result): def test_ufunc_override_normalize_signature(self): # gh-5674 class SomeClass(object): - def __array_ufunc__(self, ufunc, method, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): return kw a = SomeClass() @@ -3086,52 +3087,56 @@ def __array_ufunc__(self, ufunc, method, inputs, **kw): def test_array_ufunc_index(self): # Check that index is set appropriately, also if only an output # is passed on (latter is another regression tests for github bug 4753) + # This also checks implicitly that 'out' is always a tuple. class CheckIndex(object): - def __array_ufunc__(self, ufunc, method, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): for i, a in enumerate(inputs): if a is self: return i - return None + # calls below mean we must be in an output. + for j, a in enumerate(kw['out']): + if a is self: + return (j,) a = CheckIndex() dummy = np.arange(2.) # 1 input, 1 output assert_equal(np.sin(a), 0) - assert_equal(np.sin(dummy, a), None) - assert_equal(np.sin(dummy, out=a), None) - assert_equal(np.sin(dummy, out=(a,)), None) + assert_equal(np.sin(dummy, a), (0,)) + assert_equal(np.sin(dummy, out=a), (0,)) + assert_equal(np.sin(dummy, out=(a,)), (0,)) assert_equal(np.sin(a, a), 0) assert_equal(np.sin(a, out=a), 0) assert_equal(np.sin(a, out=(a,)), 0) # 1 input, 2 outputs - assert_equal(np.modf(dummy, a), None) - assert_equal(np.modf(dummy, None, a), None) - assert_equal(np.modf(dummy, dummy, a), None) - assert_equal(np.modf(dummy, out=a), None) - assert_equal(np.modf(dummy, out=(a,)), None) - assert_equal(np.modf(dummy, out=(a, None)), None) - assert_equal(np.modf(dummy, out=(a, dummy)), None) - assert_equal(np.modf(dummy, out=(None, a)), None) - assert_equal(np.modf(dummy, out=(dummy, a)), None) + assert_equal(np.modf(dummy, a), (0,)) + assert_equal(np.modf(dummy, None, a), (1,)) + assert_equal(np.modf(dummy, dummy, a), (1,)) + assert_equal(np.modf(dummy, out=a), (0,)) + assert_equal(np.modf(dummy, out=(a,)), (0,)) + assert_equal(np.modf(dummy, out=(a, None)), (0,)) + assert_equal(np.modf(dummy, out=(a, dummy)), (0,)) + assert_equal(np.modf(dummy, out=(None, a)), (1,)) + assert_equal(np.modf(dummy, out=(dummy, a)), (1,)) assert_equal(np.modf(a, out=(dummy, a)), 0) # 2 inputs, 1 output assert_equal(np.add(a, dummy), 0) assert_equal(np.add(dummy, a), 1) - assert_equal(np.add(dummy, dummy, a), None) + assert_equal(np.add(dummy, dummy, a), (0,)) assert_equal(np.add(dummy, a, a), 1) - assert_equal(np.add(dummy, dummy, out=a), None) - assert_equal(np.add(dummy, dummy, out=(a,)), None) + assert_equal(np.add(dummy, dummy, out=a), (0,)) + assert_equal(np.add(dummy, dummy, out=(a,)), (0,)) assert_equal(np.add(a, dummy, out=a), 0) def test_out_override(self): # regression test for github bug 4753 class OutClass(np.ndarray): - def __array_ufunc__(self, ufunc, method, inputs, **kw): + def __array_ufunc__(self, ufunc, method, *inputs, **kw): if 'out' in kw: tmp_kw = kw.copy() tmp_kw.pop('out') func = getattr(ufunc, method) - kw['out'][...] = func(*inputs, **tmp_kw) + kw['out'][0][...] = func(*inputs, **tmp_kw) A = np.array([0]).view(OutClass) B = np.array([5]) @@ -5272,14 +5277,14 @@ class B(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return "B" class C(np.ndarray): def __new__(cls, *args, **kwargs): return np.array(*args, **kwargs).view(cls) - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return NotImplemented class D(np.ndarray): diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 3776db84e4f1..1d0518f88417 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1225,7 +1225,7 @@ def test_structured_equal(self): # https://github.com/numpy/numpy/issues/4855 class MyA(np.ndarray): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return getattr(ufunc, method)(*(input.view(np.ndarray) for input in inputs), **kwargs) a = np.arange(12.).reshape(4,3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 3d4f12aeb7ca..53bbf507bbae 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1571,7 +1571,7 @@ def __array__(self): def test_ufunc_override(self): class A(object): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return self, func, method, inputs, kwargs a = A() @@ -1605,23 +1605,23 @@ def quatro_mul(a, b, c, d): four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) class A(object): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "A" class ASub(A): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "ASub" class B(object): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return "B" class C(object): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented class CSub(object): - def __array_ufunc__(self, func, method, inputs, **kwargs): + def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented a = A() @@ -1685,7 +1685,7 @@ def __array_ufunc__(self, func, method, inputs, **kwargs): def test_ufunc_override_methods(self): class A(object): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return self, ufunc, method, inputs, kwargs # __call__ @@ -1704,9 +1704,9 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'reduce') assert_equal(res[3], (a,)) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) # reduce, kwargs res = np.multiply.reduce(a, axis='axis0', dtype='dtype0', out='out0', @@ -1716,9 +1716,9 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'reduce') assert_equal(res[3], (a,)) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'keepdims': 'keep0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'keepdims': 'keep0', + 'axis': 'axis0'}) # accumulate, pos args res = np.multiply.accumulate(a, 'axis0', 'dtype0', 'out0') @@ -1727,8 +1727,8 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'accumulate') assert_equal(res[3], (a,)) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'axis': 'axis0'}) # accumulate, kwargs res = np.multiply.accumulate(a, axis='axis0', dtype='dtype0', @@ -1738,8 +1738,8 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'accumulate') assert_equal(res[3], (a,)) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'axis': 'axis0'}) # reduceat, pos args res = np.multiply.reduceat(a, [4, 2], 'axis0', 'dtype0', 'out0') @@ -1748,8 +1748,8 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'reduceat') assert_equal(res[3], (a, [4, 2])) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'axis': 'axis0'}) # reduceat, kwargs res = np.multiply.reduceat(a, [4, 2], axis='axis0', dtype='dtype0', @@ -1759,8 +1759,8 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): assert_equal(res[2], 'reduceat') assert_equal(res[3], (a, [4, 2])) assert_equal(res[4], {'dtype':'dtype0', - 'out': 'out0', - 'axis': 'axis0'}) + 'out': ('out0',), + 'axis': 'axis0'}) # outer res = np.multiply.outer(a, 42) @@ -1780,11 +1780,11 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): def test_ufunc_override_out(self): class A(object): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs class B(object): - def __array_ufunc__(self, ufunc, method, inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return kwargs a = A() @@ -1796,12 +1796,12 @@ def __array_ufunc__(self, ufunc, method, inputs, **kwargs): res4 = np.multiply(a, 4, 'out_arg') res5 = np.multiply(a, 5, out='out_arg') - assert_equal(res0['out'], 'out_arg') - assert_equal(res1['out'], 'out_arg') - assert_equal(res2['out'], 'out_arg') - assert_equal(res3['out'], 'out_arg') - assert_equal(res4['out'], 'out_arg') - assert_equal(res5['out'], 'out_arg') + assert_equal(res0['out'][0], 'out_arg') + assert_equal(res1['out'][0], 'out_arg') + assert_equal(res2['out'][0], 'out_arg') + assert_equal(res3['out'][0], 'out_arg') + assert_equal(res4['out'][0], 'out_arg') + assert_equal(res5['out'][0], 'out_arg') # ufuncs with multiple output modf and frexp. res6 = np.modf(a, 'out0', 'out1') From 6a3ca31530b67c91742240c0b8ee59903f4ec49f Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Tue, 14 Mar 2017 18:27:33 -0400 Subject: [PATCH 11/43] DOC: describe current implementation of __array_ufunc__. --- doc/source/reference/arrays.classes.rst | 98 +++++++++++++------------ numpy/doc/subclassing.py | 4 + 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index b1fb95d91684..9a6aeb89385b 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -39,74 +39,70 @@ Special attributes and methods NumPy provides several hooks that classes can customize: -.. method:: class.__array_ufunc__(ufunc, method, inputs, **kwargs) +.. method:: class.__array_ufunc__(ufunc, method, *inputs, **kwargs) - .. versionadded:: 1.11 + .. versionadded:: 1.13 - Any class (ndarray subclass or not) can define this method to - override behavior of NumPy's ufuncs. This works quite similarly to - Python's ``__mul__`` and other binary operation routines. + Any class (ndarray subclass or not) can define this method (or set it to + :obj:`None`) to override behavior of NumPy's ufuncs. This works quite + similarly to Python's ``__mul__`` and other binary operation routines. - *ufunc* is the ufunc object that was called. - *method* is a string indicating which Ufunc method was called (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, ``"accumulate"``, ``"outer"``, ``"inner"``). - - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *inputs* is a tuple of the input arguments to the ``ufunc``. - *kwargs* is a dictionary containing the optional input arguments - of the ufunc. The ``out`` argument is always contained in - *kwargs*, if given. See the discussion in :ref:`ufuncs` for - details. + of the ufunc. If given, any ``out`` arguments, both positional + and keyword, are passed as a :obj:`tuple` in *kwargs*. See the + discussion in :ref:`ufuncs` for details. The method should return either the result of the operation, or - :obj:`NotImplemented` if the operation requested is not - implemented. + :obj:`NotImplemented` if the operation requested is not implemented. If one of the arguments has a :func:`__array_ufunc__` method, it is executed *instead* of the ufunc. If more than one of the input arguments implements :func:`__array_ufunc__`, they are tried in the order: subclasses before superclasses, otherwise left to right. The - first routine returning something else than :obj:`NotImplemented` + first routine returning something other than :obj:`NotImplemented` determines the result. If all of the :func:`__array_ufunc__` operations return :obj:`NotImplemented`, a :exc:`TypeError` is raised. - If an :class:`ndarray` subclass defines the :func:`__array_ufunc__` - method, this disables the :func:`__array_wrap__`, - :func:`__array_prepare__`, :data:`__array_priority__` mechanism - described below. - .. note:: In addition to ufuncs, :func:`__array_ufunc__` also - overrides the behavior of :func:`numpy.dot` even though it is - not an Ufunc. - - .. note:: If you also define right-hand binary operator override - methods (such as ``__rmul__``) or comparison operations (such as - ``__gt__``) in your class, they take precedence over the - :func:`__array_ufunc__` mechanism when resolving results of - binary operations (such as ``ndarray_obj * your_obj``). - - The technical special case is: ``ndarray.__mul__`` returns - ``NotImplemented`` if the other object is *not* a subclass of - :class:`ndarray`, and defines both ``__array_ufunc__`` and - ``__rmul__``. Similar exception applies for the other operations - than multiplication. - - In such a case, when computing a binary operation such as - ``ndarray_obj * your_obj``, your ``__array_ufunc__`` method - *will not* be called. Instead, the execution passes on to your - right-hand ``__rmul__`` operation, as per standard Python - operator override rules. - - Similar special case applies to *in-place operations*: If you - define ``__rmul__``, then ``ndarray_obj *= your_obj`` *will not* - call your ``__array_ufunc__`` implementation. Instead, the - default Python behavior ``ndarray_obj = ndarray_obj * your_obj`` - occurs. - - Note that the above discussion applies only to Python's builtin - binary operation mechanism. ``np.multiply(ndarray_obj, - your_obj)`` always calls only your ``__array_ufunc__``, as - expected. + overrides the behavior of :func:`numpy.dot` and :func:`numpy.matmul` + even though these are not ufuncs. + + Like with other methods in python, such as ``__hash__`` and + ``__iter__``, it is possible to indicate that your class does *not* + support ufuncs by setting ``__array_ufunc__ = None``. With this, + inside ufuncs, your class will be treated as if it returned + :obj:`NotImplemented` (which will lead to an :exc:`TypeError` + unless another class also provides a :func:`__array_ufunc__` method + which knows what to do with your class). + + The presence of :func:`__array_ufunc__` also influences how binary + and comparison operators are dealt with, such as ``__add__``, + ``__gt__``, etc. If it is not :obj:`None`, the assumption is that + your code can handle such operations via the ufunc mechanism, and + hence forward methods on :class:`ndarray` will call the ufuncs + unconditionally (i.e., even if your class has defined reverse + methods such as ``__radd__``, ``__le__``, etc.). If + ``__array_ufunc__ = None``, however, forward methods on + :class:`ndarray` will unconditionally return :obj:`NotImplemented`, + so that your reverse methods will get called. + + .. note:: If you subclass :class:`ndarray`, we strongly recommend + that you avoid confusion by neither setting + :func:`__array_ufunc__` to :obj:`None` (which seems to make no + sense for an array subclass), nor defining it and also defining + reverse methods (which will be called by ``CPython`` in + preference over the :class:`ndarray` forward methods). + + .. note:: If a class defines the :func:`__array_ufunc__` method, + this disables the :func:`__array_wrap__`, + :func:`__array_prepare__`, :data:`__array_priority__` mechanism + described below (which may eventually be deprecated). .. method:: class.__array_finalize__(obj) @@ -129,6 +125,9 @@ NumPy provides several hooks that classes can customize: the subclass and update metadata before returning the array to the ufunc for computation. + .. note:: It is hoped to eventually deprecate this method in favour of + :func:__array_ufunc__`. + .. method:: class.__array_wrap__(array, context=None) At the end of every :ref:`ufunc `, this method @@ -148,6 +147,9 @@ NumPy provides several hooks that classes can customize: possibility for the Python type of the returned object. Subclasses inherit a default value of 0.0 for this attribute. + .. note:: It is hoped to eventually deprecate this attribute in favour + of :func:__array_ufunc__`. + .. method:: class.__array__([dtype]) If a class (ndarray subclass or not) having the :func:`__array__` diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index b6c742a2b175..ea6de2ccfad3 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -511,6 +511,10 @@ def __array_wrap__(self, arr, context=None): Like ``__array_wrap__``, ``__array_prepare__`` must return an ndarray or subclass thereof or raise an error. +.. note:: As of numpy 1.13, there also is a new, more powerful method to + handle how a subclass deals with ufuncs, ``__array_ufunc__``. For details, + see the reference section. + Extra gotchas - custom ``__del__`` methods and ndarray.base ----------------------------------------------------------- From 79bb7331457bc8e3696935ba3fb29d53791330a6 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Thu, 23 Mar 2017 15:14:13 -0600 Subject: [PATCH 12/43] DOC: Style and sphinx fixes for arrays.classes.rst. --- doc/source/reference/arrays.classes.rst | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 9a6aeb89385b..f068bbcc74de 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -39,13 +39,13 @@ Special attributes and methods NumPy provides several hooks that classes can customize: -.. method:: class.__array_ufunc__(ufunc, method, *inputs, **kwargs) +.. py:method:: class.__array_ufunc__(ufunc, method, *inputs, **kwargs) .. versionadded:: 1.13 - Any class (ndarray subclass or not) can define this method (or set it to - :obj:`None`) to override behavior of NumPy's ufuncs. This works quite - similarly to Python's ``__mul__`` and other binary operation routines. + Any class, ndarray subclass or not, can define this method or set it to + :obj:`None` in order to override the behavior of NumPy's ufuncs. This works + quite similarly to Python's ``__mul__`` and other binary operation routines. - *ufunc* is the ufunc object that was called. - *method* is a string indicating which Ufunc method was called @@ -94,17 +94,17 @@ NumPy provides several hooks that classes can customize: .. note:: If you subclass :class:`ndarray`, we strongly recommend that you avoid confusion by neither setting - :func:`__array_ufunc__` to :obj:`None` (which seems to make no - sense for an array subclass), nor defining it and also defining - reverse methods (which will be called by ``CPython`` in - preference over the :class:`ndarray` forward methods). + :func:`__array_ufunc__` to :obj:`None`, which makes no + sense for an array subclass, nor by defining it and also defining + reverse methods, which methods will be called by ``CPython`` in + preference over the :class:`ndarray` forward methods. .. note:: If a class defines the :func:`__array_ufunc__` method, this disables the :func:`__array_wrap__`, :func:`__array_prepare__`, :data:`__array_priority__` mechanism described below (which may eventually be deprecated). -.. method:: class.__array_finalize__(obj) +.. py:method:: class.__array_finalize__(obj) This method is called whenever the system internally allocates a new array from *obj*, where *obj* is a subclass (subtype) of the @@ -113,7 +113,7 @@ NumPy provides several hooks that classes can customize: to update meta-information from the "parent." Subclasses inherit a default implementation of this method that does nothing. -.. method:: class.__array_prepare__(array, context=None) +.. py:method:: class.__array_prepare__(array, context=None) At the beginning of every :ref:`ufunc `, this method is called on the input object with the highest array @@ -126,9 +126,9 @@ NumPy provides several hooks that classes can customize: ufunc for computation. .. note:: It is hoped to eventually deprecate this method in favour of - :func:__array_ufunc__`. + :func:`__array_ufunc__`. -.. method:: class.__array_wrap__(array, context=None) +.. py:method:: class.__array_wrap__(array, context=None) At the end of every :ref:`ufunc `, this method is called on the input object with the highest array priority, or @@ -140,7 +140,7 @@ NumPy provides several hooks that classes can customize: into an instance of the subclass and update metadata before returning the array to the user. -.. data:: class.__array_priority__ +.. py:attribute:: class.__array_priority__ The value of this attribute is used to determine what type of object to return in situations where there is more than one @@ -148,9 +148,9 @@ NumPy provides several hooks that classes can customize: inherit a default value of 0.0 for this attribute. .. note:: It is hoped to eventually deprecate this attribute in favour - of :func:__array_ufunc__`. + of :func:`__array_ufunc__`. -.. method:: class.__array__([dtype]) +.. py:method:: class.__array__([dtype]) If a class (ndarray subclass or not) having the :func:`__array__` method is used as the output object of an :ref:`ufunc From 7c3dc5aacd78b21ae31e7510c622129f95eac8de Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Fri, 24 Mar 2017 20:49:36 -0400 Subject: [PATCH 13/43] TST: test that gufuncs are also overridden by __array_ufunc__. --- numpy/core/tests/test_umath.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 53bbf507bbae..21e0a649ed5d 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1821,6 +1821,29 @@ def __array_ufunc__(self, *a, **kwargs): for func in [np.divide, np.dot]: assert_raises(ValueError, func, a, a) + def test_gufunc_override(self): + # gufunc are just ufunc instances, but follow a different path, + # so check __array_ufunc__ overrides them properly. + class A(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self, ufunc, method, inputs, kwargs + + a = A() + res = np.core.umath_tests.inner1d(a, a) + assert_equal(res[0], a) + assert_equal(res[1], np.core.umath_tests.inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (a, a)) + assert_equal(res[4], {}) + + res = np.core.umath_tests.inner1d(1, 1, out=a) + assert_equal(res[0], a) + assert_equal(res[1], np.core.umath_tests.inner1d) + assert_equal(res[2], '__call__') + assert_equal(res[3], (1, 1)) + assert_equal(res[4], {'out': (a,)}) + + class TestChoose(TestCase): def test_mixed(self): c = np.array([True, True]) From 71201d286c38157f25325096286b2bc6089fbf84 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Tue, 14 Mar 2017 22:56:44 -0400 Subject: [PATCH 14/43] DOC: Describe __array_func__ in subclassing This includes the use of super everywhere, and in the brief description of __array_ufunc__ in the reference section. --- doc/source/reference/arrays.classes.rst | 34 +++-- numpy/doc/subclassing.py | 181 +++++++++++++++++++----- 2 files changed, 173 insertions(+), 42 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index f068bbcc74de..387ac2de1897 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -43,6 +43,10 @@ NumPy provides several hooks that classes can customize: .. versionadded:: 1.13 + .. note:: The API is `provisional + `_, + i.e., we do not yet guarantee backward compatibility. + Any class, ndarray subclass or not, can define this method or set it to :obj:`None` in order to override the behavior of NumPy's ufuncs. This works quite similarly to Python's ``__mul__`` and other binary operation routines. @@ -70,8 +74,11 @@ NumPy provides several hooks that classes can customize: raised. .. note:: In addition to ufuncs, :func:`__array_ufunc__` also - overrides the behavior of :func:`numpy.dot` and :func:`numpy.matmul` - even though these are not ufuncs. + overrides the behavior of :func:`numpy.dot` and :func:`numpy.matmul`. + This even though these are not ufuncs, but they can be thought of as + :ref:`generalized universal functions` + (which are overridden). We intend to extend this behaviour to other + relevant functions. Like with other methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* @@ -92,17 +99,26 @@ NumPy provides several hooks that classes can customize: :class:`ndarray` will unconditionally return :obj:`NotImplemented`, so that your reverse methods will get called. - .. note:: If you subclass :class:`ndarray`, we strongly recommend - that you avoid confusion by neither setting - :func:`__array_ufunc__` to :obj:`None`, which makes no - sense for an array subclass, nor by defining it and also defining - reverse methods, which methods will be called by ``CPython`` in - preference over the :class:`ndarray` forward methods. + .. note:: If you subclass :class:`ndarray`: + + - We strongly recommend that you avoid confusion by neither setting + :func:`__array_ufunc__` to :obj:`None`, which makes no sense for + an array subclass, nor by defining it and also defining reverse + methods, which methods will be called by ``CPython`` in + preference over the :class:`ndarray` forward methods. + - :class:`ndarray` defines its own :func:`__array_ufunc__`, which + corresponds to ``getattr(ufunc, method)(*inputs, **kwargs)``. Hence, + a typical override of :func:`__array_ufunc__` would convert any + instances of one's own class, pass these on to its superclass using + ``super().__array_ufunc__(*inputs, **kwargs)``, and finally return + the results after possible back-conversion. This practice ensures + that it is possible to have a hierarchy of subclasses. See + :ref:`Subclassing ndarray ` for details. .. note:: If a class defines the :func:`__array_ufunc__` method, this disables the :func:`__array_wrap__`, :func:`__array_prepare__`, :data:`__array_priority__` mechanism - described below (which may eventually be deprecated). + described below for ufuncs (which may eventually be deprecated). .. py:method:: class.__array_finalize__(obj) diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index ea6de2ccfad3..3e16ae87056a 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -1,5 +1,4 @@ -""" -============================= +"""============================= Subclassing ndarray in python ============================= @@ -220,8 +219,9 @@ class other than the class in which it is defined, the ``__init__`` * For the explicit constructor call, our subclass will need to create a new ndarray instance of its own class. In practice this means that we, the authors of the code, will need to make a call to - ``ndarray.__new__(MySubClass,...)``, or do view casting of an existing - array (see below) + ``ndarray.__new__(MySubClass,...)``, a class-hierarchy prepared call to + ``super(MySubClass, cls).__new__(cls, ...)``, or do view casting of an + existing array (see below) * For view casting and new-from-template, the equivalent of ``ndarray.__new__(MySubClass,...`` is called, at the C level. @@ -237,7 +237,7 @@ class other than the class in which it is defined, the ``__init__`` class C(np.ndarray): def __new__(cls, *args, **kwargs): print('In __new__ with class %s' % cls) - return np.ndarray.__new__(cls, *args, **kwargs) + return super(C, cls).__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): # in practice you probably will not need or want an __init__ @@ -275,7 +275,8 @@ def __array_finalize__(self, obj): def __array_finalize__(self, obj): -``ndarray.__new__`` passes ``__array_finalize__`` the new object, of our +One sees that the ``super`` call, which goes to +``ndarray.__new__``, passes ``__array_finalize__`` the new object, of our own class (``self``) as well as the object from which the view has been taken (``obj``). As you can see from the output above, the ``self`` is always a newly created instance of our subclass, and the type of ``obj`` @@ -303,13 +304,14 @@ def __array_finalize__(self, obj): class InfoArray(np.ndarray): def __new__(subtype, shape, dtype=float, buffer=None, offset=0, - strides=None, order=None, info=None): + strides=None, order=None, info=None): # Create the ndarray instance of our type, given the usual # ndarray input arguments. This will call the standard # ndarray constructor, but return an object of our type. # It also triggers a call to InfoArray.__array_finalize__ - obj = np.ndarray.__new__(subtype, shape, dtype, buffer, offset, strides, - order) + obj = super(InfoArray, subtype).__new__(subtype, shape, dtype, + buffer, offset, strides, + order) # set the new 'info' attribute to the value passed obj.info = info # Finally, we must return the newly created object: @@ -412,15 +414,132 @@ def __array_finalize__(self, obj): >>> v.info 'information' -.. _array-wrap: +.. _array-ufunc: + +``__array_ufunc__`` for ufuncs +------------------------------ + + .. versionadded:: 1.13 + +A subclass can override what happens when executing numpy ufuncs on it by +overriding the default ``ndarray.__array_ufunc__`` method. This method is +executed *instead* of the ufunc and should return either the result of the +operation, or :obj:`NotImplemented` if the operation requested is not +implemented. + +The signature of ``__array_ufunc__`` is:: + + def __array_ufunc__(ufunc, method, *inputs, **kwargs): -``__array_wrap__`` for ufuncs -------------------------------------------------------- + - *ufunc* is the ufunc object that was called. + - *method* is a string indicating which Ufunc method was called + (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, + ``"accumulate"``, ``"outer"``, ``"inner"``). + - *inputs* is a tuple of the input arguments to the ``ufunc``. + - *kwargs* is a dictionary containing the optional input arguments + of the ufunc. If given, any ``out`` arguments, both positional + and keyword, are passed as a :obj:`tuple` in *kwargs*. -``__array_wrap__`` gets called at the end of numpy ufuncs and other numpy -functions, to allow a subclass to set the type of the return value -and update attributes and metadata. Let's show how this works with an example. -First we make the same subclass as above, but with a different name and +A typical implementation would convert any inputs or ouputs that are +instances of one's own class, pass everything on to a superclass using +``super()``, and finally return the results after possible +back-conversion. An example, taken from the test case +``test_ufunc_override_with_super`` in ``core/tests/test_umath.pu``, is the +following. + +.. testcode:: + + input numpy as np + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', []) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + + info = {key: no for (key, no) in (('inputs', in_no), + ('outputs', out_no)) + if no != []} + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if not isinstance(results, tuple): + if not isinstance(results, np.ndarray): + return results + results = (results,) + + if outputs == []: + outputs = [None] * len(results) + results = tuple(result.view(A) if output is None else output + for result, output in zip(results, outputs)) + if isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + +So, this class does not actually do anything interesting: it just +converts any instances of its own to regular ndarray (otherwise, we'd +get infinite recursion!), and adds an ``info`` dictionary that tells +which inputs and outputs it converted. Hence, e.g., + +>>> a = np.arange(5.).view(A) +>>> b = np.sin(a) +>>> b.info +{'inputs': [0]} +>>> b = np.sin(np.arange(5.), out=(a,)) +>>> b.info +{'outputs': [0]} +>>> a = np.arange(5.).view(A) +>>> b = np.ones(1).view(A) +>>> a += b +>>> a.info +{'inputs': [0, 1], 'outputs': [0]} + +Note that one might also consider just doing ``getattr(ufunc, +methods)(*inputs, **kwargs)`` instead of the ``super`` call. This would +work (indeed, ``ndarray.__array_ufunc__`` effectively does just that), but +by using ``super`` one can more easily have a class hierarchy. E.g., +suppose we had another class ``B`` that defined ``__array_ufunc__`` and +then made a subclass ``C`` depending on both, i.e., ``class C(A, B)`` +without yet another ``__array_ufunc__`` override. Then any ufunc on an +instance of ``C`` would pass on to ``A.__array_ufunc__``, the ``super`` +call in ``A`` would go to ``B.__array_ufunc__``, and the ``super`` call in +``B`` would go to ``ndarray.__array_ufunc__``. + +.. _array-wrap: + +``__array_wrap__`` for ufuncs and other functions +------------------------------------------------- + +Prior to numpy 1.13, the behaviour of ufuncs could be tuned using +``__array_wrap__`` and ``__array_prepare__``. These two allowed one to +change the output type of a ufunc, but, in constrast to +``__array_ufunc__``, did not allow one to make any changes to the inputs. +It is hoped to eventually deprecate these, but ``__array_wrap__`` is also +used by other numpy functions and methods, such as ``squeeze``, so at the +present time is still needed for full functionality. + +Conceptually, ``__array_wrap__`` "wraps up the action" in the sense of +allowing a subclass to set the type of the return value and update +attributes and metadata. Let's show how this works with an example. First +we return to the simpler example subclass, but with a different name and some print statements: .. testcode:: @@ -446,7 +565,7 @@ def __array_wrap__(self, out_arr, context=None): print(' self is %s' % repr(self)) print(' arr is %s' % repr(out_arr)) # then just call the parent - return np.ndarray.__array_wrap__(self, out_arr, context) + return super(MySubClass, self).__array_wrap__(self, out_arr, context) We run a ufunc on an instance of our new array: @@ -467,13 +586,12 @@ def __array_wrap__(self, out_arr, context=None): >>> ret.info 'spam' -Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method of the -input with the highest ``__array_priority__`` value, in this case -``MySubClass.__array_wrap__``, with arguments ``self`` as ``obj``, and -``out_arr`` as the (ndarray) result of the addition. In turn, the -default ``__array_wrap__`` (``ndarray.__array_wrap__``) has cast the -result to class ``MySubClass``, and called ``__array_finalize__`` - -hence the copying of the ``info`` attribute. This has all happened at the C level. +Note that the ufunc (``np.add``) has called the ``__array_wrap__`` method +with arguments ``self`` as ``obj``, and ``out_arr`` as the (ndarray) result +of the addition. In turn, the default ``__array_wrap__`` +(``ndarray.__array_wrap__``) has cast the result to class ``MySubClass``, +and called ``__array_finalize__`` - hence the copying of the ``info`` +attribute. This has all happened at the C level. But, we could do anything we wanted: @@ -494,11 +612,12 @@ def __array_wrap__(self, arr, context=None): So, by defining a specific ``__array_wrap__`` method for our subclass, we can tweak the output from ufuncs. The ``__array_wrap__`` method requires ``self``, then an argument - which is the result of the ufunc - -and an optional parameter *context*. This parameter is returned by some -ufuncs as a 3-element tuple: (name of the ufunc, argument of the ufunc, -domain of the ufunc). ``__array_wrap__`` should return an instance of -its containing class. See the masked array subclass for an -implementation. +and an optional parameter *context*. This parameter is returned by +ufuncs as a 3-element tuple: (name of the ufunc, arguments of the ufunc, +domain of the ufunc), but is not set by other numpy functions. Though, +as seen above, it is possible to do otherwise, ``__array_wrap__`` should +return an instance of its containing class. See the masked array +subclass for an implementation. In addition to ``__array_wrap__``, which is called on the way out of the ufunc, there is also an ``__array_prepare__`` method which is called on @@ -511,10 +630,6 @@ def __array_wrap__(self, arr, context=None): Like ``__array_wrap__``, ``__array_prepare__`` must return an ndarray or subclass thereof or raise an error. -.. note:: As of numpy 1.13, there also is a new, more powerful method to - handle how a subclass deals with ufuncs, ``__array_ufunc__``. For details, - see the reference section. - Extra gotchas - custom ``__del__`` methods and ndarray.base ----------------------------------------------------------- From 30417109170d1f5f1256172e6506ea32751b0587 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sun, 12 Mar 2017 23:05:17 -0400 Subject: [PATCH 15/43] ENH: implement ndarray.__array_ufunc__ --- numpy/core/src/multiarray/methods.c | 42 +++++++++++++ numpy/core/src/private/ufunc_override.c | 64 +++++++++++++++++--- numpy/core/tests/test_multiarray.py | 9 ++- numpy/core/tests/test_umath.py | 80 +++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 12 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index dc02b29f4959..36f48ce8f1e3 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1007,6 +1007,45 @@ array_getarray(PyArrayObject *self, PyObject *args) } +static PyObject * +array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *ufunc, *method_name, *normal_args, *ufunc_method, *result; + + if (PyTuple_Size(args) < 2) { + PyErr_SetString(PyExc_TypeError, + "__array_ufunc__ requires at least 2 arguments"); + return NULL; + } + ufunc = PyTuple_GET_ITEM(args, 0); + if (ufunc == NULL) { + return NULL; + } + + method_name = PyTuple_GET_ITEM(args, 1); + if (method_name == NULL) { + return NULL; + } + + normal_args = PyTuple_GetSlice(args, 2, PyTuple_GET_SIZE(args)); + if (normal_args == NULL) { + return NULL; + } + + ufunc_method = PyObject_GetAttr(ufunc, method_name); + if (ufunc_method == NULL) { + Py_DECREF(normal_args); + return NULL; + } + + result = PyObject_Call(ufunc_method, normal_args, kwds); + Py_DECREF(normal_args); + Py_DECREF(ufunc_method); + /* no need to DECREF borrowed references ufunc and method_name */ + return result; +} + + static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) { @@ -2471,6 +2510,9 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { {"__array_wrap__", (PyCFunction)array_wraparray, METH_VARARGS, NULL}, + {"__array_ufunc__", + (PyCFunction)array_ufunc, + METH_VARARGS | METH_KEYWORDS, NULL}, /* for the sys module */ {"__sizeof__", diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index 655aa27f7f94..a7d71ee94584 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -1,7 +1,10 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + #include "npy_pycompat.h" #include "numpy/ufuncobject.h" #include "get_attr_string.h" +#include "npy_import.h" #include "ufunc_override.h" @@ -169,6 +172,47 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, return; } +/* + * Check whether an object has __array_ufunc__ defined on its class and it + * is not the default, i.e., the object is not an ndarray, and its + * __array_ufunc__ is not the same as that of ndarray. + * + * Note that since this module is used with both multiarray and umath, we do + * not have access to PyArray_Type and therewith neither to PyArray_CheckExact + * nor to the default __array_ufunc__ method, so instead we import locally. + * TODO: Can this really not be done more smartly? + */ +static int +has_non_default_array_ufunc(PyObject *obj) +{ + static PyObject *ndarray = NULL; + static PyObject *ndarray_array_ufunc = NULL; + PyObject *cls_array_ufunc; + int non_default; + + /* on first entry, import and cache ndarray and its __array_ufunc__ */ + if (ndarray == NULL) { + npy_cache_import("numpy.core.multiarray", "ndarray", &ndarray); + ndarray_array_ufunc = PyObject_GetAttrString(ndarray, + "__array_ufunc__"); + } + + /* Fast return for ndarray */ + if ((PyObject *)Py_TYPE(obj) == ndarray) { + return 0; + } + /* does the class define __array_ufunc__? */ + cls_array_ufunc = PyArray_GetAttrString_SuppressException( + (PyObject *)Py_TYPE(obj), "__array_ufunc__"); + if (cls_array_ufunc == NULL) { + return 0; + } + /* is it different from ndarray.__array_ufunc__? */ + non_default = (cls_array_ufunc != ndarray_array_ufunc); + Py_DECREF(cls_array_ufunc); + return non_default; +} + /* * Check a set of args for the `__array_ufunc__` method. If more than one of * the input arguments implements `__array_ufunc__`, they are tried in the @@ -194,7 +238,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, int out_kwd_is_tuple = 0; int noa = 0; /* Number of overriding args.*/ - PyObject *tmp; PyObject *obj; PyObject *out_kwd_obj = NULL; PyObject *other_obj; @@ -203,9 +246,9 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ PyObject *normal_kwds = NULL; + PyObject *override_args = NULL; PyObject *with_override[NPY_MAXARGS]; Py_ssize_t len; - PyObject *override_args; /* * Check inputs @@ -250,11 +293,14 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, obj = out_kwd_obj; } } - tmp = PyArray_GetAttrString_SuppressException(obj, "__array_ufunc__"); - if (tmp) { - Py_DECREF(tmp); - with_override[noa] = obj; - ++noa; + /* + * Now see if the object provides an __array_ufunc__. However, we should + * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as + * any ndarray subclass instances that did not override __array_ufunc__. + */ + if (has_non_default_array_ufunc(obj)) { + with_override[noa] = obj; + ++noa; } } @@ -282,7 +328,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, out = PyDict_GetItemString(normal_kwds, "out"); if (out != NULL) { if (PyTuple_Check(out)) { - int all_none; + int all_none = 1; int i; for (i = 0; i < PyTuple_GET_SIZE(out); i++) { @@ -362,7 +408,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, Py_INCREF(ufunc); /* PyTuple_SET_ITEM steals reference */ - PyTuple_SET_ITEM(override_args, 0, ufunc); + PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); Py_INCREF(method_name); PyTuple_SET_ITEM(override_args, 1, method_name); for (i = 0; i < len; i++) { diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 9766ee47af90..5da66da0c36a 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2988,7 +2988,8 @@ def norm(result): "__array_ufunc__") else: if (isinstance(obj, np.ndarray) and - not hasattr(obj, "__array_ufunc__")): + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): # __array__ gets ignored res = norm(arr_method(obj)) assert_(res.__class__ is obj.__class__) @@ -3004,7 +3005,8 @@ def norm(result): assert_equal(res[1], ufunc) else: if (isinstance(obj, np.ndarray) and - not hasattr(obj, "__array_ufunc__")): + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): # __array__ gets ignored res = norm(arr_rmethod(obj)) assert_(res.__class__ is obj.__class__) @@ -3025,7 +3027,8 @@ def norm(result): assert_(res[-1]["out"][0] is arr) else: if (isinstance(obj, np.ndarray) and - not hasattr(obj, "__array_ufunc__")): + (type(obj).__array_ufunc__ is + np.ndarray.__array_ufunc__)): # __array__ gets ignored assert_(arr_imethod(obj) is arr) else: diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 21e0a649ed5d..c7d7781be79a 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1843,6 +1843,86 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_equal(res[3], (1, 1)) assert_equal(res[4], {'out': (a,)}) + def test_ufunc_override_with_super(self): + + class A(np.ndarray): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + args = [] + in_no = [] + for i, input_ in enumerate(inputs): + if isinstance(input_, A): + in_no.append(i) + args.append(input_.view(np.ndarray)) + else: + args.append(input_) + + outputs = kwargs.pop('out', []) + out_no = [] + if outputs: + out_args = [] + for j, output in enumerate(outputs): + if isinstance(output, A): + out_no.append(j) + out_args.append(output.view(np.ndarray)) + else: + out_args.append(output) + kwargs['out'] = tuple(out_args) + + info = {key: no for (key, no) in (('inputs', in_no), + ('outputs', out_no)) + if no != []} + + results = super(A, self).__array_ufunc__(ufunc, method, + *args, **kwargs) + if not isinstance(results, tuple): + if not isinstance(results, np.ndarray): + return results + results = (results,) + + if outputs == []: + outputs = [None] * len(results) + results = tuple(result.view(A) if output is None else output + for result, output in zip(results, outputs)) + if isinstance(results[0], A): + results[0].info = info + + return results[0] if len(results) == 1 else results + + d = np.arange(5.) + a = np.arange(5.).view(A) + # 1 input, 1 output + b = np.sin(a) + check = np.sin(d) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0]}) + b = np.sin(d, out=(a,)) + assert_(np.all(check == b)) + assert_equal(b.info, {'outputs': [0]}) + assert_(b is a) + a = np.arange(5.).view(A) + b = np.sin(a, out=a) + assert_(np.all(check == b)) + assert_equal(b.info, {'inputs': [0], 'outputs': [0]}) + # 1 input, 2 outputs + a = np.arange(5.).view(A) + b1, b2 = np.modf(a) + assert_equal(b1.info, {'inputs': [0]}) + b1, b2 = np.modf(d, out=(None, a)) + assert_(b2 is a) + assert_equal(b1.info, {'outputs': [1]}) + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c1, c2 = np.modf(a, out=(a, b)) + assert_(c1 is a) + assert_(c2 is b) + assert_equal(c1.info, {'inputs': [0], 'outputs': [0, 1]}) + # 2 input, 1 output + a = np.arange(5.).view(A) + b = np.arange(5.).view(A) + c = np.add(a, b, out=a) + assert_(c is a) + assert_equal(c.info, {'inputs': [0, 1], 'outputs': [0]}) + class TestChoose(TestCase): def test_mixed(self): From 5fe6fc640d752fe9e4a9a51635bf070b503aa85e Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Fri, 31 Mar 2017 12:28:16 -0400 Subject: [PATCH 16/43] DOC Update NEP to reflect actual implementation. --- doc/neps/ufunc-overrides.rst | 262 ++++++++++++++++++++--------------- 1 file changed, 151 insertions(+), 111 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 480e229c2a74..f69da0090b02 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -2,6 +2,8 @@ A Mechanism for Overriding Ufuncs ================================= +.. currentmodule:: numpy + :Author: Blake Griffith :Contact: blake.g@utexas.edu :Date: 2013-07-10 @@ -10,25 +12,32 @@ A Mechanism for Overriding Ufuncs :Author: Nathaniel Smith +:Author: Marten van Kerkwijk +:Date: 2017-03-31 Executive summary ================= NumPy's universal functions (ufuncs) currently have some limited -functionality for operating on user defined subclasses of ndarray using -``__array_prepare__`` and ``__array_wrap__`` [1]_, and there is little -to no support for arbitrary objects. e.g. SciPy's sparse matrices [2]_ -[3]_. +functionality for operating on user defined subclasses of +:class:`ndarray` using ``__array_prepare__`` and ``__array_wrap__`` +[1]_, and there is little to no support for arbitrary +objects. e.g. SciPy's sparse matrices [2]_ [3]_. Here we propose adding a mechanism to override ufuncs based on the ufunc -checking each of it's arguments for a ``__numpy_ufunc__`` method. -On discovery of ``__numpy_ufunc__`` the ufunc will hand off the +checking each of it's arguments for a ``__array_ufunc__`` method. +On discovery of ``__array_ufunc__`` the ufunc will hand off the operation to the method. This covers some of the same ground as Travis Oliphant's proposal to retro-fit NumPy with multi-methods [4]_, which would solve the same problem. The mechanism here follows more closely the way Python enables -classes to override ``__mul__`` and other binary operations. +classes to override ``__mul__`` and other binary operations. It also +specifically addresses how binary operators and ufuncs should interact. + +.. note:: In earlier iterations, the override was called + ``__numpy_ufunc__``. An implementation was made, but had not + quite the right behaviour, hence the change in name. .. [1] http://docs.python.org/doc/numpy/user/basics.subclassing.html .. [2] https://github.com/scipy/scipy/issues/2123 @@ -41,13 +50,14 @@ Motivation The current machinery for dispatching Ufuncs is generally agreed to be insufficient. There have been lengthy discussions and other proposed -solutions [5]_. +solutions [5]_, [6]_. -Using ufuncs with subclasses of ndarray is limited to ``__array_prepare__`` and -``__array_wrap__`` to prepare the arguments, but these don't allow you to for -example change the shape or the data of the arguments. Trying to ufunc things -that don't subclass ndarray is even more difficult, as the input arguments tend -to be cast to object arrays, which ends up producing surprising results. +Using ufuncs with subclasses of :class:`ndarray` is limited to +``__array_prepare__`` and ``__array_wrap__`` to prepare the arguments, +but these don't allow you to for example change the shape or the data of +the arguments. Trying to ufunc things that don't subclass +:class:`ndarray` is even more difficult, as the input arguments tend to +be cast to object arrays, which ends up producing surprising results. Take this example of ufuncs interoperability with sparse matrices.:: @@ -81,7 +91,7 @@ Take this example of ufuncs interoperability with sparse matrices.:: In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! Out[5]: NotImplemted -Returning ``NotImplemented`` to user should not happen. Moreover:: +Returning :obj:`NotImplemented` to user should not happen. Moreover:: In [6]: np.multiply(asp, b) Out[6]: array([[ <3x3 sparse matrix of type '' @@ -106,21 +116,24 @@ Returning ``NotImplemented`` to user should not happen. Moreover:: Here, it appears that the sparse matrix was converted to an object array scalar, which was then multiplied with all elements of the ``b`` array. However, this behavior is more confusing than useful, and having a -``TypeError`` would be preferable. +:exc:`TypeError` would be preferable. -Adding the ``__numpy_ufunc__`` functionality fixes this and would +Adding the ``__array_ufunc__`` functionality fixes this and would deprecate the other ufunc modifying functions. .. [5] http://mail.python.org/pipermail/numpy-discussion/2011-June/056945.html +.. [6] https://github.com/numpy/numpy/issues/5844 Proposed interface ================== -Objects that want to override Ufuncs can define a ``__numpy_ufunc__`` method. -The method signature is:: +The standard array class :class:`ndarray` gains an ``__array_ufunc__`` +method and objects can override Ufuncs by overriding this method (if +they are :class:`ndarray` subclasses) or defining their own. The method +signature is:: - def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs) + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) Here: @@ -128,141 +141,168 @@ Here: - *method* is a string indicating which Ufunc method was called (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, ``"accumulate"``, ``"outer"``, ``"inner"``). -- *i* is the index of *self* in *inputs*. - *inputs* is a tuple of the input arguments to the ``ufunc`` - *kwargs* are the keyword arguments passed to the function. The ``out`` - arguments are always contained in *kwargs*, how positional variables - are passed is discussed below. - -The ufunc's arguments are first normalized into a tuple of input data -(``inputs``), and dict of keyword arguments. If there are output -arguments they are handled as follows: - -- One positional output variable x is passed in the kwargs dict as ``out : - x``. -- Multiple positional output variables ``x0, x1, ...`` are passed as a tuple - in the kwargs dict as ``out : (x0, x1, ...)``. -- Keyword output variables like ``out = x`` and ``out = (x0, x1, ...)`` are - passed unchanged to the kwargs dict like ``out : x`` and ``out : (x0, x1, - ...)`` respectively. -- Combinations of positional and keyword output variables are not - supported. + arguments are always contained as a tuple in *kwargs*. + +Hence, the arguments are normalized: only the input data (``inputs``) +are passed on as positional arguments, all the others are passed on as a +dict of keyword arguments (``kwargs``). In particular, if there are +output arguments, positional are otherwise, they are passed on as a +tuple in the ``out`` keyword argument. The function dispatch proceeds as follows: -- If one of the input arguments implements ``__numpy_ufunc__`` it is +- If one of the input arguments implements ``__array_ufunc__`` it is executed instead of the Ufunc. -- If more than one of the input arguments implements ``__numpy_ufunc__``, +- If more than one of the input arguments implements ``__array_ufunc__``, they are tried in the following order: subclasses before superclasses, - otherwise left to right. The first ``__numpy_ufunc__`` method returning - something else than ``NotImplemented`` determines the return value of + otherwise left to right. The first ``__array_ufunc__`` method returning + something else than :obj:`NotImplemented` determines the return value of the Ufunc. -- If all ``__numpy_ufunc__`` methods of the input arguments return - ``NotImplemented``, a ``TypeError`` is raised. +- If all ``__array_ufunc__`` methods of the input arguments return + :obj:`NotImplemented`, a :exc:`TypeError` is raised. -- If a ``__numpy_ufunc__`` method raises an error, the error is propagated +- If a ``__array_ufunc__`` method raises an error, the error is propagated immediately. -If none of the input arguments has a ``__numpy_ufunc__`` method, the +If none of the input arguments has an ``__array_ufunc__`` method, the execution falls back on the default ufunc behaviour. +Subclass hierarchies +-------------------- + +Hierarchies of such containers (say, a masked quantity), are most easily +constructed if methods consistently use :func:`super` to pass through +the class hierarchy [7]_. To support this, :class:`ndarray` has its own +``__array_ufunc__`` method (which is equivalent to ``getattr(ufunc, +method)(*inputs, **kwargs)``, i.e., if any of the (adjusted) inputs +still defines ``__array_ufunc__`` that will be called in turn). This +should be particularly useful for container-like subclasses of +:class:`ndarray`, which add an attribute like a unit or mask to a +regular :class:`ndarray`. Such classes can do possible adjustment of the +arguments relevant to their own class, pass on to another class in the +hierarchy using :func:`super` until the Ufunc is actually done, and then +do possible adjustments of the outputs. + +Turning Ufuncs off +------------------ + +For some classes, Ufuncs make no sense, and, like for other special +methods [8]_, one can indicate Ufuncs are not available by setting +``__array_ufunc__`` to :obj:`None`. Inside a Ufunc, this is +equivalent to unconditionally return :obj:`NotImplemented`, and thus +will lead to a :exc:`TypeError` (unless another operand implements +``__array_ufunc__`` and knows how to deal with the class). + +.. [7] https://rhettinger.wordpress.com/2011/05/26/super-considered-super/ + +.. [8] https://docs.python.org/3/reference/datamodel.html#specialnames In combination with Python's binary operations ---------------------------------------------- -The ``__numpy_ufunc__`` mechanism is fully independent of Python's +The ``__array_ufunc__`` mechanism is fully independent of Python's standard operator override mechanism, and the two do not interact directly. -They however have indirect interactions, because NumPy's ``ndarray`` -type implements its binary operations via Ufuncs. Effectively, we have:: - - class ndarray(object): +They have indirect interactions, however, because NumPy's +:class:`ndarray` type implements its binary operations via Ufuncs. For +most numerical classes, the easiest way to override binary operations is +thus to define ``__array_ufunc__`` and override the corresponding +Ufunc. The class can then, like :class:`ndarray` itself, define the +binary operators in terms of Ufuncs. Here, one has to take some care. +E.g., the simplest implementation would be:: + + class ArrayLike(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + ... + return result ... def __mul__(self, other): - return np.multiply(self, other) + return self.__array_ufunc__(np.multiply, '__call__', self, other) -Suppose now we have a second class:: +Suppose now, however, that ``other`` is class that does not know how to +deal with arrays and ufuncs, but does know how to do multiplication:: class MyObject(object): - def __numpy_ufunc__(self, *a, **kw): - return "ufunc" + __array_ufunc__ = None def __mul__(self, other): return 1234 def __rmul__(self, other): return 4321 In this case, standard Python override rules combined with the above -discussion imply:: +discussion would imply:: - a = MyObject() - b = np.array([0]) + mine = MyObject() + arr = ArrayLike([0]) - a * b # == 1234 OK - b * a # == "ufunc" surprising + mine * arr # == 1234 OK + arr * mine # TypeError surprising -This is not what would be naively expected, and is therefore somewhat -undesirable behavior. +The reason why this would occur is: because ``MyObject`` is not an +``ArrayLike`` subclass, Python resolves the expression ``arr * mine`` by +calling first ``arr.__mul__``. In the above implementation, this would +just call the Ufunc, which would see that ``mine.__array_ufunc__`` is +:obj:`None` and raise a :exc:`TypeError`. (Note that if ``MyObject`` +is a subclass of :class:`ndarray`, Python calls ``mine.__rmul__`` first.) -The reason why this occurs is: because ``MyObject`` is not an ndarray -subclass, Python resolves the expression ``b * a`` by calling first -``b.__mul__``. Since NumPy implements this via an Ufunc, the call is -forwarded to ``__numpy_ufunc__`` and not to ``__rmul__``. Note that if -``MyObject`` is a subclass of ``ndarray``, Python calls ``a.__rmul__`` -first. The issue is therefore that ``__numpy_ufunc__`` implements -"virtual subclassing" of ndarray behavior, without actual subclassing. +So, a better implementation of the binary operators would check whether +the other class can be dealt with in ``__array_ufunc__`` and, if not, +return :obj:`NotImplemented`:: -This issue can be resolved by a modification of the binary operation -methods in NumPy:: - - class ndarray(object): + class ArrayLike(object): ... def __mul__(self, other): - if (not isinstance(other, self.__class__) - and hasattr(other, '__numpy_ufunc__') - and hasattr(other, '__rmul__')): - return NotImplemented - return np.multiply(self, other) - - def __imul__(self, other): - if (other.__class__ is not self.__class__ - and hasattr(other, '__numpy_ufunc__') - and hasattr(other, '__rmul__')): + if getattr(other, '__array_ufunc__', False) is None: return NotImplemented - return np.multiply(self, other, out=self) - - b * a # == 4321 OK - -The rationale here is the following: since the user class explicitly -defines both ``__numpy_ufunc__`` and ``__rmul__``, the implementor has -very likely made sure that the ``__rmul__`` method can process ndarrays. -If not, the special case is simple to deal with (just call -``np.multiply``). - -The exclusion of subclasses of self can be made because Python itself -calls the right-hand method first in this case. Moreover, it is -desirable that ndarray subclasses are able to inherit the right-hand -binary operation methods from ndarray. - -The same priority shuffling needs to be done also for the in-place -operations, so that ``MyObject.__rmul__`` is prioritized over -``ndarray.__imul__``. - + return self.__array_ufunc__(np.multiply, '__call__', self, other) + + arr = ArrayLike([0]) + + arr * mine # == 4321 OK + +Indeed, after long discussion about whether it might make more sense to +ask classes like ``ArrayLike`` to implement a full ``__array_ufunc__`` +[6]_, the same design as the above was agreed on for :class:`ndarray` +itself. + +.. note:: The above holds for regular operators. For in-place + operators, :class:`ndarray` never returns + :obj:`NotImplemented`, i.e., ``ndarr *= mine`` would always + lead to a :exc:`TypeError`. This is because for arrays + in-place operations cannot generically be replaced by a simple + reverse operation. For instance, sticking to the above + example, what would ``ndarr[:] *= mine`` imply? Assuming it + means ``ndarr[:] = ndarr[:] * mine``, as python does by + default, is likely to be wrong. + +Extension to other numpy functions +---------------------------------- + +The ``__array_ufunc__`` method is used to override :func:`~numpy.dot` +and :func:`~numpy.matmul` as well, since while these functions are not +(yet) implemented as (generalized) Ufuncs, they are very similar. For +other functions, such as :func:`~numpy.median`, :func:`~numpy.min`, +etc., implementations as (generalized) Ufuncs may well be possible and +logical as well, in which case it will become possible to override these +as well. Demo ==== -A pull request[6]_ has been made including the changes proposed in this NEP. -Here is a demo highlighting the functionality.:: +A pull request [8]_ has been made including the changes and revisions +proposed in this NEP. Here is a demo highlighting the functionality.:: In [1]: import numpy as np; In [2]: a = np.array([1]) In [3]: class B(): - ...: def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + ...: def __array_ufunc__(self, func, method, pos, inputs, **kwargs): ...: return "B" ...: @@ -274,24 +314,24 @@ Here is a demo highlighting the functionality.:: In [6]: np.multiply(a, b) Out[6]: 'B' -A simple ``__numpy_ufunc__`` has been added to SciPy's sparse matrices -Currently this only handles ``np.dot`` and ``np.multiply`` because it was the -two most common cases where users would attempt to use sparse matrices with ufuncs. -The method is defined below:: +As a simple example, one could add the following ``__array_ufunc__`` to +SciPy's sparse matrices (just for ``np.dot`` and ``np.multiply`` as +these are the two most common cases where users would attempt to use +sparse matrices with ufuncs):: - def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + def __array_ufunc__(self, func, method, pos, inputs, **kwargs): """Method for compatibility with NumPy's ufuncs and dot functions. """ without_self = list(inputs) - del without_self[pos] + without_self.pop(self) without_self = tuple(without_self) - if func == np.multiply: + if func is np.multiply: return self.multiply(*without_self) - elif func == np.dot: + elif func is np.dot: if pos == 0: return self.__mul__(inputs[1]) if pos == 1: From e0928235fac6ac20db6b263d438607f4df90ae46 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sun, 2 Apr 2017 11:54:38 -0400 Subject: [PATCH 17/43] MAINT: let ndarray.__array_ufunc__ bail if any overrides are in place. As part of this, split off the checking for overrides from the actual trying to execute them (in ufunc_override.c), so that just the check can be done in ndarray.__array_ufunc__. --- numpy/core/src/multiarray/methods.c | 35 ++++++---- numpy/core/src/private/ufunc_override.c | 87 ++++++++++++++++--------- numpy/core/src/private/ufunc_override.h | 4 ++ 3 files changed, 82 insertions(+), 44 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 36f48ce8f1e3..6cfd05cd6634 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1010,37 +1010,48 @@ array_getarray(PyArrayObject *self, PyObject *args) static PyObject * array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) { - PyObject *ufunc, *method_name, *normal_args, *ufunc_method, *result; + PyObject *ufunc, *method_name, *normal_args, *ufunc_method; + PyObject *result = NULL; if (PyTuple_Size(args) < 2) { PyErr_SetString(PyExc_TypeError, "__array_ufunc__ requires at least 2 arguments"); return NULL; } + normal_args = PyTuple_GetSlice(args, 2, PyTuple_GET_SIZE(args)); + if (normal_args == NULL) { + return NULL; + } + /* ndarray cannot handle overrides itself */ + if (PyUFunc_HasOverride(normal_args, kwds, NULL)) { + result = Py_NotImplemented; + Py_INCREF(Py_NotImplemented); + goto cleanup; + } + ufunc = PyTuple_GET_ITEM(args, 0); if (ufunc == NULL) { - return NULL; + goto cleanup; } method_name = PyTuple_GET_ITEM(args, 1); if (method_name == NULL) { - return NULL; - } - - normal_args = PyTuple_GetSlice(args, 2, PyTuple_GET_SIZE(args)); - if (normal_args == NULL) { - return NULL; + goto cleanup; } + /* + * TODO(?): call into UFunc code at a later point, since here arguments are + * already normalized and we do not have to look for __array_ufunc__ again. + */ ufunc_method = PyObject_GetAttr(ufunc, method_name); if (ufunc_method == NULL) { - Py_DECREF(normal_args); - return NULL; + goto cleanup; } - result = PyObject_Call(ufunc_method, normal_args, kwds); - Py_DECREF(normal_args); Py_DECREF(ufunc_method); + +cleanup: + Py_DECREF(normal_args); /* no need to DECREF borrowed references ufunc and method_name */ return result; } diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index a7d71ee94584..1db4e54b979a 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -214,24 +214,16 @@ has_non_default_array_ufunc(PyObject *obj) } /* - * Check a set of args for the `__array_ufunc__` method. If more than one of - * the input arguments implements `__array_ufunc__`, they are tried in the - * order: subclasses before superclasses, otherwise left to right. The first - * (non-None) routine returning something other than `NotImplemented` - * determines the result. If all of the `__array_ufunc__` operations return - * `NotImplemented` (or are None), a `TypeError` is raised. - * - * Returns 0 on success and 1 on exception. On success, *result contains the - * result of the operation, if any. If *result is NULL, there is no override. + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Returns the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL). + * returns -1 on failure. */ NPY_NO_EXPORT int -PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result, - int nin) +PyUFunc_HasOverride(PyObject *args, PyObject *kwds, + PyObject **with_override) { int i; - int j; int nargs; int nout_kwd = 0; @@ -240,16 +232,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *obj; PyObject *out_kwd_obj = NULL; - PyObject *other_obj; - - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ - PyObject *normal_kwds = NULL; - - PyObject *override_args = NULL; - PyObject *with_override[NPY_MAXARGS]; - Py_ssize_t len; - /* * Check inputs */ @@ -266,7 +248,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, "to PyUFunc_CheckOverride"); goto fail; } - /* be sure to include possible 'out' keyword argument. */ if (kwds && PyDict_CheckExact(kwds)) { out_kwd_obj = PyDict_GetItemString(kwds, "out"); @@ -287,7 +268,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } else { if (out_kwd_is_tuple) { - obj = PyTuple_GET_ITEM(out_kwd_obj, i-nargs); + obj = PyTuple_GET_ITEM(out_kwd_obj, i - nargs); } else { obj = out_kwd_obj; @@ -299,22 +280,60 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, * any ndarray subclass instances that did not override __array_ufunc__. */ if (has_non_default_array_ufunc(obj)) { - with_override[noa] = obj; + if (with_override != NULL) { + with_override[noa] = obj; + } ++noa; } } + return noa; +fail: + return -1; +} +/* + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. + * + * Returns 0 on success and 1 on exception. On success, *result contains the + * result of the operation, if any. If *result is NULL, there is no override. + */ +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result, + int nin) +{ + int i; + int j; + + int noa; + PyObject *with_override[NPY_MAXARGS]; + + PyObject *obj; + PyObject *other_obj; + + PyObject *method_name = NULL; + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + + PyObject *override_args = NULL; + Py_ssize_t len; + + /* + * Check inputs for overrides + */ + noa = PyUFunc_HasOverride(args, kwds, with_override); /* No overrides, bail out.*/ if (noa == 0) { *result = NULL; return 0; } - method_name = PyUString_FromString(method); - if (method_name == NULL) { - goto fail; - } - /* * Normalize ufunc arguments. */ @@ -409,6 +428,10 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, Py_INCREF(ufunc); /* PyTuple_SET_ITEM steals reference */ PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); + method_name = PyUString_FromString(method); + if (method_name == NULL) { + goto fail; + } Py_INCREF(method_name); PyTuple_SET_ITEM(override_args, 1, method_name); for (i = 0; i < len; i++) { diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index ee18e569c34d..db5e84fd5871 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -4,6 +4,10 @@ #include "npy_config.h" #include "numpy/ufuncobject.h" +NPY_NO_EXPORT int +PyUFunc_HasOverride(PyObject *args, PyObject *kwds, + PyObject **with_override); + NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, From 114789495535bf9243d2935977aed60fdc842203 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 1 Apr 2017 17:02:46 +0200 Subject: [PATCH 18/43] MAINT: Update array_ufunc NEP. Bring into compliance with current ndarray.__array_ufunc__ implementation and type casting hierarchy. --- doc/neps/ufunc-overrides.rst | 177 +++++++++++++++++++++++++++++++---- doc/source/conf.py | 1 + 2 files changed, 159 insertions(+), 19 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index f69da0090b02..c994ce9bf8ff 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -171,21 +171,148 @@ The function dispatch proceeds as follows: If none of the input arguments has an ``__array_ufunc__`` method, the execution falls back on the default ufunc behaviour. + +Type casting hierarchy +---------------------- + +Similarly to the Python operator dispatch mechanism, writing ufunc +dispatch methods requires some discipline in order to achieve +predictable results. + +In particular, it is useful to maintain a clear idea of what types can +be upcast to others, possibly indirectly (i.e. A->B->C is implemented +but direct A->C not). Moreover, one should make sure the implementations of +``__array_ufunc__``, which implicitly define the type casting hierarchy, +don't contradict this. + +The following rules should be followed: + +1. The ``__array_ufunc__`` for type A should either return + `NotImplemented`, or return an output of type A (unless an + ``out=`` argument was given, in which case ``out`` is returned). + +2. For any two different types *A*, *B*, the relation "A can handle B" + defined as:: + + a.__array_ufunc__(..., b, ...) is not NotImplemented + + for instances *a* and *b* of *A* and *B*, defines the + edges B->A of a graph. + + This graph must be a directed acyclic graph. + +Under these conditions, the transitive closure of the "can handle" +relation defines a strict partial ordering of the types -- that is, the +type casting hierarchy. + +In other words, for any given class A, all other classes that define +``__array_ufunc__`` must belong to exactly one of the groups: + +- *Above A*: their ``__array_ufunc__`` can handle class A or some + member of the "above A" classes. In other words, these are the types + that A can be (indirectly) upcast to in ufuncs. + +- *Below A*: they can be handled by the ``__array_ufunc__`` of class A + or the ``__array_ufunc__`` of some member of the "below A" classes. In + other words, these are the types that can be (indirectly) upcast to A + in ufuncs. + +- *Incompatible*: neither above nor below A; types for which no + (indirect) upcasting is possible. + +This guarantees that expressions involving ufuncs either raise a +`TypeError`, or the result type is independent of what ufuncs were +called, what order they were called in, and what order their arguments +were in. Moreover, which ``__array_ufunc__`` payload code runs at each +step is independent of the order of arguments of the ufuncs. + +Note also that while converting inputs that don't have +``__array_ufunc__`` to `ndarray` via `np.asarray` is consistent with the +type casting hierarchy, also returning `NotImplemented` is +consistent. However, the numpy ufunc (legacy) behavior is to try to +convert unknown objects to ndarrays. + + +.. admonition:: Example + + Type casting hierarchy + + .. graphviz:: + + digraph array_ufuncs { + rankdir=BT; + A -> C; + B -> C; + D -> B; + ndarray -> A; + ndarray -> B; + } + + The ``__array_ufunc__`` of type A can handle ndarrays, B can handle ndarray and D, + and C can handle A and B but not ndarrays or D. The resulting graph is a DAG, + and defines a type casting hierarchy, with relations ``C > A > + ndarray``, ``C > B > ndarray``, ``C > B > D``. The type B is incompatible + relative to A and vice versa, and A and ndarray are incompatible relative to D. + Ufunc expressions involving these classes produce results of the highest type + involved or raise a TypeError. + + Subclass hierarchies -------------------- -Hierarchies of such containers (say, a masked quantity), are most easily -constructed if methods consistently use :func:`super` to pass through -the class hierarchy [7]_. To support this, :class:`ndarray` has its own -``__array_ufunc__`` method (which is equivalent to ``getattr(ufunc, -method)(*inputs, **kwargs)``, i.e., if any of the (adjusted) inputs -still defines ``__array_ufunc__`` that will be called in turn). This -should be particularly useful for container-like subclasses of -:class:`ndarray`, which add an attribute like a unit or mask to a -regular :class:`ndarray`. Such classes can do possible adjustment of the -arguments relevant to their own class, pass on to another class in the -hierarchy using :func:`super` until the Ufunc is actually done, and then -do possible adjustments of the outputs. +Generally, it is desirable to mirror the class hierarchy in the ufunc +type casting hierarchy. The recommendation is that an +``__array_ufunc__`` implementation of a class should generally return +`NotImplemented` unless the inputs are instances of the same class or +superclasses. This guarantees that in the type casting hierarchy, +superclasses are below, subclasses above, and other classes are +incompatible. Exceptions to this need to check they respect the +implicit type casting hierarchy. + +Subclasses can be easily constructed if methods consistently use +:func:`super` to pass through the class hierarchy [7]_. To support +this, :class:`ndarray` has its own ``__array_ufunc__`` method, +equivalent to:: + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.pop('out', None) + out_tuple = out if out is not None else () + + # Handle items of type(self), superclasses, and items + # without __array_ufunc__. Bail out in other cases. + items = [] + for item in inputs + out_tuple: + if isinstance(self, type(item)) or not hasattr(item, '__array_ufunc__'): + # Cast to plain ndarrays + items.append(np.asarray(item)) + else: + return NotImplemented + + # Perform ufunc on the underlying ndarrays (no __array_ufunc__ dispatch) + result = getattr(ufunc, method)(*items, **kwargs) + + # Cast output to type(self), unless `out` specified + if out is not None: + return result + + if isinstance(result, tuple): + return tuple(x.view(type(self)) for x in result) + else: + return result.view(type(self)) + +Note that, as a special case, the ufunc dispatch mechanism does not call +the `__array_ufunc__` method for inputs of `ndarray` type. As a +consequence, calling `ndarray.__array_ufunc__` will not result to a +nested ufunc dispatch cycle. Custom implementations of +`__array_ufunc__` should generally avoid nested dispatch cycles. + +This should be particularly useful for subclasses of :class:`ndarray`, +which only add an attribute like a unit or mask to a regular +:class:`ndarray`. In their `__array_ufunc__` implementation, such +classes can do possible adjustment of the arguments relevant to their +own class, and pass on to superclass implementation using :func:`super` +until the ufunc is actually done, and then do possible adjustments of +the outputs. Turning Ufuncs off ------------------ @@ -193,9 +320,12 @@ Turning Ufuncs off For some classes, Ufuncs make no sense, and, like for other special methods [8]_, one can indicate Ufuncs are not available by setting ``__array_ufunc__`` to :obj:`None`. Inside a Ufunc, this is -equivalent to unconditionally return :obj:`NotImplemented`, and thus +equivalent to unconditionally returning :obj:`NotImplemented`, and thus will lead to a :exc:`TypeError` (unless another operand implements -``__array_ufunc__`` and knows how to deal with the class). +``__array_ufunc__`` and specifically knows how to deal with the class). + +In the type casting hierarchy, this makes the type incompatible relative +to `ndarray`. .. [7] https://rhettinger.wordpress.com/2011/05/26/super-considered-super/ @@ -217,10 +347,11 @@ binary operators in terms of Ufuncs. Here, one has to take some care. E.g., the simplest implementation would be:: class ArrayLike(object): + ... def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... return result - ... + def __mul__(self, other): return self.__array_ufunc__(np.multiply, '__call__', self, other) @@ -229,20 +360,28 @@ deal with arrays and ufuncs, but does know how to do multiplication:: class MyObject(object): __array_ufunc__ = None + def __init__(self, value): + self.value = value + def __repr__(self): + return "MyObject({!r})".format(self.value) def __mul__(self, other): - return 1234 + return MyObject(1234) def __rmul__(self, other): - return 4321 + return MyObject(4321) In this case, standard Python override rules combined with the above discussion would imply:: - mine = MyObject() + mine = MyObject(0) arr = ArrayLike([0]) - mine * arr # == 1234 OK + mine * arr # == MyObject(1234) OK arr * mine # TypeError surprising +XXX: but it doesn't raise a TypeError, because `__mul__` calls +directly `__array_ufunc__`, which sees the `__array_ufunc__ == None`, and +bails out with `NotImplemented`? + The reason why this would occur is: because ``MyObject`` is not an ``ArrayLike`` subclass, Python resolves the expression ``arr * mine`` by calling first ``arr.__mul__``. In the above implementation, this would diff --git a/doc/source/conf.py b/doc/source/conf.py index 8c18e423a1ed..2bafc50ebfe5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -22,6 +22,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc', 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', 'sphinx.ext.doctest', 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', 'matplotlib.sphinxext.plot_directive'] # Add any paths that contain templates here, relative to this directory. From e325a10a18ed1d9f2a20672c4874d8646fb7a2b2 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 2 Apr 2017 00:58:49 +0200 Subject: [PATCH 19/43] DOC: Document behavior of ufuncs with default ndarray.__array_ufunc__ --- doc/neps/ufunc-overrides.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index c994ce9bf8ff..4cf7bd757a20 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -153,23 +153,30 @@ tuple in the ``out`` keyword argument. The function dispatch proceeds as follows: -- If one of the input arguments implements ``__array_ufunc__`` it is - executed instead of the Ufunc. +- If an input argument has a ``__array_ufunc__`` attribute, but its + value is ``ndarray.__array_ufunc__``, the attribute is considered to + be absent in what follows. This happens for instances of `ndarray` + and those `ndarray` subclasses that did not override their inherited + ``__array_ufunc__`` implementation. + +- If one of the input arguments implements ``__array_ufunc__``, it is + executed instead of the ufunc. - If more than one of the input arguments implements ``__array_ufunc__``, they are tried in the following order: subclasses before superclasses, - otherwise left to right. The first ``__array_ufunc__`` method returning - something else than :obj:`NotImplemented` determines the return value of - the Ufunc. + otherwise left to right. + +- The first ``__array_ufunc__`` method returning something else than + :obj:`NotImplemented` determines the return value of the Ufunc. - If all ``__array_ufunc__`` methods of the input arguments return :obj:`NotImplemented`, a :exc:`TypeError` is raised. -- If a ``__array_ufunc__`` method raises an error, the error is propagated - immediately. +- If a ``__array_ufunc__`` method raises an error, the error is + propagated immediately. -If none of the input arguments has an ``__array_ufunc__`` method, the -execution falls back on the default ufunc behaviour. +- If none of the input arguments had an ``__array_ufunc__`` method, the + execution falls back on the default ufunc behaviour. Type casting hierarchy @@ -301,7 +308,8 @@ equivalent to:: return result.view(type(self)) Note that, as a special case, the ufunc dispatch mechanism does not call -the `__array_ufunc__` method for inputs of `ndarray` type. As a +this `ndarray.__array_ufunc__` method, even for `ndarray` subclasses +if they have not overridden the default `ndarray` implementation. As a consequence, calling `ndarray.__array_ufunc__` will not result to a nested ufunc dispatch cycle. Custom implementations of `__array_ufunc__` should generally avoid nested dispatch cycles. From 39c2273e03070e8682f91852667286fe6f7d436b Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 2 Apr 2017 16:43:23 +0200 Subject: [PATCH 20/43] DOC: Update ndarray.__array_ufunc__ documentation vs. review comments --- doc/neps/ufunc-overrides.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 4cf7bd757a20..95342906a880 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -282,24 +282,21 @@ this, :class:`ndarray` has its own ``__array_ufunc__`` method, equivalent to:: def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - out = kwargs.pop('out', None) - out_tuple = out if out is not None else () - # Handle items of type(self), superclasses, and items # without __array_ufunc__. Bail out in other cases. - items = [] - for item in inputs + out_tuple: + for item in inputs: if isinstance(self, type(item)) or not hasattr(item, '__array_ufunc__'): - # Cast to plain ndarrays - items.append(np.asarray(item)) + pass else: return NotImplemented # Perform ufunc on the underlying ndarrays (no __array_ufunc__ dispatch) + items = [np.asarray(item) if isinstance(item, np.ndarray) else item + for item in inputs] result = getattr(ufunc, method)(*items, **kwargs) # Cast output to type(self), unless `out` specified - if out is not None: + if kwargs['out']: return result if isinstance(result, tuple): From 6b41d110b092ae32252253b4d0a54d40b5628b93 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sun, 2 Apr 2017 14:00:34 -0400 Subject: [PATCH 21/43] DOC: clarify use of super and getattr --- doc/source/reference/arrays.classes.rst | 84 ++++++++++++++++++------- numpy/doc/subclassing.py | 47 ++++++++++---- 2 files changed, 98 insertions(+), 33 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 387ac2de1897..1ece99af613f 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -64,14 +64,14 @@ NumPy provides several hooks that classes can customize: The method should return either the result of the operation, or :obj:`NotImplemented` if the operation requested is not implemented. - If one of the arguments has a :func:`__array_ufunc__` method, it is - executed *instead* of the ufunc. If more than one of the input + If one of the input or output arguments has a :func:`__array_ufunc__` + method, it is executed *instead* of the ufunc. If more than one of the arguments implements :func:`__array_ufunc__`, they are tried in the - order: subclasses before superclasses, otherwise left to right. The - first routine returning something other than :obj:`NotImplemented` - determines the result. If all of the :func:`__array_ufunc__` - operations return :obj:`NotImplemented`, a :exc:`TypeError` is - raised. + order: subclasses before superclasses, inputs before outputs, otherwise + left to right. The first routine returning something other than + :obj:`NotImplemented` determines the result. If all of the + :func:`__array_ufunc__` operations return :obj:`NotImplemented`, a + :exc:`TypeError` is raised. .. note:: In addition to ufuncs, :func:`__array_ufunc__` also overrides the behavior of :func:`numpy.dot` and :func:`numpy.matmul`. @@ -80,7 +80,7 @@ NumPy provides several hooks that classes can customize: (which are overridden). We intend to extend this behaviour to other relevant functions. - Like with other methods in python, such as ``__hash__`` and + Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* support ufuncs by setting ``__array_ufunc__ = None``. With this, inside ufuncs, your class will be treated as if it returned @@ -99,20 +99,60 @@ NumPy provides several hooks that classes can customize: :class:`ndarray` will unconditionally return :obj:`NotImplemented`, so that your reverse methods will get called. - .. note:: If you subclass :class:`ndarray`: - - - We strongly recommend that you avoid confusion by neither setting - :func:`__array_ufunc__` to :obj:`None`, which makes no sense for - an array subclass, nor by defining it and also defining reverse - methods, which methods will be called by ``CPython`` in - preference over the :class:`ndarray` forward methods. - - :class:`ndarray` defines its own :func:`__array_ufunc__`, which - corresponds to ``getattr(ufunc, method)(*inputs, **kwargs)``. Hence, - a typical override of :func:`__array_ufunc__` would convert any - instances of one's own class, pass these on to its superclass using - ``super().__array_ufunc__(*inputs, **kwargs)``, and finally return - the results after possible back-conversion. This practice ensures - that it is possible to have a hierarchy of subclasses. See + The presence of :func:`__array_ufunc__` also influences how + :class:`ndarray` handles binary operations like ``arr + obj`` and ``arr + < obj`` when ``arr`` is an :class:`ndarray` and ``obj`` is an instance + of a custom class. There are two possibilities. If + ``obj.__array_ufunc__`` is present and not :obj:`None`, then + ``ndarray.__add__`` and friends will delegate to the ufunc machinery, + meaning that ``arr + obj`` becomes ``np.add(arr, obj)``, and then + :func:`~numpy.add` invokes ``obj.__array_ufunc__``. This is useful if you + want to define an object that acts like an array. + + Alternatively, if ``obj.__array_ufunc__`` is set to :obj:`None`, then as a + special case, special methods like ``ndarray.__add__`` will notice this + and *unconditionally* return :obj:`NotImplemented`, so that Python will + dispatch to ``obj.__radd__`` instead. This is useful if you want to define + a special object that interacts with arrays via binary operations, but + is not itself an array. For example, a units handling system might have + an object ``m`` representing the "meters" unit, and want to support the + syntax ``arr * m`` to represent that the array has units of "meters", but + not want to otherwise interact with arrays via ufuncs or otherwise. This + can be done by setting ``__array_ufunc__ = None`` and defining ``__mul__`` + and ``__rmul__`` methods. (Note that this means that writing an + ``__array_ufunc__`` that always returns :obj:`NotImplemented` is not + quite the same as setting ``__array_ufunc__ = None``: in the former + case, ``arr + obj`` will raise :exc:`TypeError`, while in the latter + case it is possible to define a ``__radd__`` method to prevent this.) + + The above does not hold for in-place operators, for which :class:`ndarray` + never returns :obj:`NotImplemented`. Hence, ``arr += obj`` would always + lead to a :exc:`TypeError`. This is because for arrays in-place operations + cannot generically be replaced by a simple reverse operation. (For + instance, by default, ``arr[:] += obj`` would be translated to ``arr[:] = + arr[:] + obj``, which would likely be wrong.) + + .. note:: If you define ``__array_ufunc__``: + + - If you are not a subclass of :class:`ndarray`, we recommend your + class define special methods like ``__add__`` and ``__lt__`` that + delegate to ufuncs just like ndarray does. We hope to provide a + helper mixin class for this. + - If you subclass :class:`ndarray`, we strongly recommend that you + avoid confusion by neither setting :func:`__array_ufunc__` to + :obj:`None`, which makes no sense for an array subclass, nor by + defining it and also defining reverse methods, which methods will + be called by ``CPython`` in preference over the :class:`ndarray` + forward methods. + - :class:`ndarray` defines its own :func:`__array_ufunc__`, which, + evaluates the ufunc if no arguments have overrides, and returns + :obj:`NotImplemented` otherwise. This may be useful for subclasses + for which :func:`__array_ufunc__` converts any instances of its own + class to :class:`ndarray`: it can then pass these on to its + superclass using ``super().__array_ufunc__(*inputs, **kwargs)``, + and finally return the results after possible back-conversion. The + advantage of this practice is that it ensures that it is possible + to have a hierarchy of subclasses that extend the behaviour. See :ref:`Subclassing ndarray ` for details. .. note:: If a class defines the :func:`__array_ufunc__` method, diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 3e16ae87056a..c42d5e330254 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -480,6 +480,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): results = super(A, self).__array_ufunc__(ufunc, method, *args, **kwargs) + if results is NotImplemented: + return NotImplemented + if not isinstance(results, tuple): if not isinstance(results, np.ndarray): return results @@ -508,27 +511,49 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): {'outputs': [0]} >>> a = np.arange(5.).view(A) >>> b = np.ones(1).view(A) +>>> c = a + b +>>> c.info +{'inputs': [0, 1]} >>> a += b >>> a.info {'inputs': [0, 1], 'outputs': [0]} -Note that one might also consider just doing ``getattr(ufunc, -methods)(*inputs, **kwargs)`` instead of the ``super`` call. This would -work (indeed, ``ndarray.__array_ufunc__`` effectively does just that), but -by using ``super`` one can more easily have a class hierarchy. E.g., -suppose we had another class ``B`` that defined ``__array_ufunc__`` and -then made a subclass ``C`` depending on both, i.e., ``class C(A, B)`` -without yet another ``__array_ufunc__`` override. Then any ufunc on an -instance of ``C`` would pass on to ``A.__array_ufunc__``, the ``super`` -call in ``A`` would go to ``B.__array_ufunc__``, and the ``super`` call in -``B`` would go to ``ndarray.__array_ufunc__``. +Note that another approach would be to to use ``getattr(ufunc, +methods)(*inputs, **kwargs)`` instead of the ``super`` call. For this example, +the result would be identical, but there is a difference if another operand +also defines ``__array_ufunc__``. E.g., lets assume that we evalulate +``np.add(a, b)``, where ``b`` is an instance of another class ``B`` that has +an override. If you use ``super`` as in the example, +``ndarray.__array_ufunc__`` will notice that ``b`` has an override, which +means it cannot evaluate the result itself. Thus, it will return +`NotImplemented` and so will our class ``A``. Then, control will be passed +over to ``b``, which either knows how to deal with us and produces a result, +or does not and returns `NotImplemented`, raising a ``TypeError``. + +If instead, we replace our ``super`` call with ``getattr(ufunc, method)``, we +effectively do ``np.add(a.view(np.ndarray), b)``. Again, ``B.__array_ufunc__`` +will be called, but now it sees an ``ndarray`` as the other argument. Likely, +it will know how to handle this, and return a new instance of the ``B`` class +to us. Our example class is not set up to handle this, but it might well be +the best approach if, e.g., one were to re-implement ``MaskedArray`` using + ``__array_ufunc__``. + +As a final note: if the ``super`` route is suited to a given class, an +advantage of using it is that it helps in constructing class hierarchies. +E.g., suppose that our other class ``B`` also used the ``super`` in its +``__array_ufunc__`` implementation, and we created a class ``C`` that depended +on both, i.e., ``class C(A, B)`` (with, for simplicity, not another +``__array_ufunc__`` override). Then any ufunc on an instance of ``C`` would +pass on to ``A.__array_ufunc__``, the ``super`` call in ``A`` would go to +``B.__array_ufunc__``, and the ``super`` call in ``B`` would go to +``ndarray.__array_ufunc__``, thus allowing ``A`` and ``B`` to collaborate. .. _array-wrap: ``__array_wrap__`` for ufuncs and other functions ------------------------------------------------- -Prior to numpy 1.13, the behaviour of ufuncs could be tuned using +Prior to numpy 1.13, the behaviour of ufuncs could only be tuned using ``__array_wrap__`` and ``__array_prepare__``. These two allowed one to change the output type of a ufunc, but, in constrast to ``__array_ufunc__``, did not allow one to make any changes to the inputs. From 0ede0e9c2619c7540d44311a3979a0890e1c3ec9 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sun, 2 Apr 2017 17:31:08 -0400 Subject: [PATCH 22/43] DOC: update NEP again. --- doc/neps/ufunc-overrides.rst | 346 ++++++++++++++++++++--------------- 1 file changed, 202 insertions(+), 144 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 95342906a880..2801802a9638 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -34,10 +34,9 @@ retro-fit NumPy with multi-methods [4]_, which would solve the same problem. The mechanism here follows more closely the way Python enables classes to override ``__mul__`` and other binary operations. It also specifically addresses how binary operators and ufuncs should interact. - -.. note:: In earlier iterations, the override was called - ``__numpy_ufunc__``. An implementation was made, but had not - quite the right behaviour, hence the change in name. +(Note that in earlier iterations, the override was called +``__numpy_ufunc__``. An implementation was made, but had not quite the +right behaviour, hence the change in name.) .. [1] http://docs.python.org/doc/numpy/user/basics.subclassing.html .. [2] https://github.com/scipy/scipy/issues/2123 @@ -53,7 +52,7 @@ insufficient. There have been lengthy discussions and other proposed solutions [5]_, [6]_. Using ufuncs with subclasses of :class:`ndarray` is limited to -``__array_prepare__`` and ``__array_wrap__`` to prepare the arguments, +``__array_prepare__`` and ``__array_wrap__`` to prepare the output arguments, but these don't allow you to for example change the shape or the data of the arguments. Trying to ufunc things that don't subclass :class:`ndarray` is even more difficult, as the input arguments tend to @@ -153,18 +152,12 @@ tuple in the ``out`` keyword argument. The function dispatch proceeds as follows: -- If an input argument has a ``__array_ufunc__`` attribute, but its - value is ``ndarray.__array_ufunc__``, the attribute is considered to - be absent in what follows. This happens for instances of `ndarray` - and those `ndarray` subclasses that did not override their inherited - ``__array_ufunc__`` implementation. - -- If one of the input arguments implements ``__array_ufunc__``, it is - executed instead of the ufunc. +- If one of the input or output arguments implements + ``__array_ufunc__``, it is executed instead of the ufunc. - If more than one of the input arguments implements ``__array_ufunc__``, they are tried in the following order: subclasses before superclasses, - otherwise left to right. + inputs before outputs, otherwise left to right. - The first ``__array_ufunc__`` method returning something else than :obj:`NotImplemented` determines the return value of the Ufunc. @@ -178,6 +171,12 @@ The function dispatch proceeds as follows: - If none of the input arguments had an ``__array_ufunc__`` method, the execution falls back on the default ufunc behaviour. +In the above, there is one proviso: if a class has an +``__array_ufunc__`` attribute but it is identical to +``ndarray.__array_ufunc__``, the attribute is ignored. This happens for +instances of `ndarray` and for `ndarray` subclasses that did not +override their inherited ``__array_ufunc__`` implementation. + Type casting hierarchy ---------------------- @@ -192,28 +191,10 @@ but direct A->C not). Moreover, one should make sure the implementations of ``__array_ufunc__``, which implicitly define the type casting hierarchy, don't contradict this. -The following rules should be followed: - -1. The ``__array_ufunc__`` for type A should either return - `NotImplemented`, or return an output of type A (unless an - ``out=`` argument was given, in which case ``out`` is returned). - -2. For any two different types *A*, *B*, the relation "A can handle B" - defined as:: - - a.__array_ufunc__(..., b, ...) is not NotImplemented - - for instances *a* and *b* of *A* and *B*, defines the - edges B->A of a graph. - - This graph must be a directed acyclic graph. - -Under these conditions, the transitive closure of the "can handle" -relation defines a strict partial ordering of the types -- that is, the -type casting hierarchy. - -In other words, for any given class A, all other classes that define -``__array_ufunc__`` must belong to exactly one of the groups: +It is useful to think of the typecasting hierarchy as a graph (see +example below) in which, for any given class A, all other classes that +define ``__array_ufunc__`` must belong to exactly one of three groups +(making this an directed acyclic graph): - *Above A*: their ``__array_ufunc__`` can handle class A or some member of the "above A" classes. In other words, these are the types @@ -227,18 +208,31 @@ In other words, for any given class A, all other classes that define - *Incompatible*: neither above nor below A; types for which no (indirect) upcasting is possible. -This guarantees that expressions involving ufuncs either raise a -`TypeError`, or the result type is independent of what ufuncs were -called, what order they were called in, and what order their arguments -were in. Moreover, which ``__array_ufunc__`` payload code runs at each -step is independent of the order of arguments of the ufuncs. +Given this grouping, to ensure that expressions involving ufuncs either +raise a :exc:`TypeError`, or have a result type that is independent of +what ufuncs were called, what order they were called in, and what order +their arguments were in, the above implies that ``__array_ufunc__`` for +type A should: + +- Return an object of type A if all other arguments are of types below A. -Note also that while converting inputs that don't have -``__array_ufunc__`` to `ndarray` via `np.asarray` is consistent with the -type casting hierarchy, also returning `NotImplemented` is -consistent. However, the numpy ufunc (legacy) behavior is to try to -convert unknown objects to ndarrays. +- Return :obj:`NotImplemented` if any argument has a type that is above + A or with which it is incompatible. +Note that there are, as always, exceptions. For instance, for a +quantity class, the results of most ufuncs should be quantities, but +this is not the case for comparison operators. For those, a quantity +class would return a plain array. + +Note also that the legacy behaviour of numpy ufunc (legacy) behavior is +to try to convert unknown objects to :class:`ndarray` via +:func:`np.asarray`. This is equivalent to placing :class:`ndarray` at +the very top of the graph, and is thus a consistent type +hierarchy (although one that causes the problems that motivate +this NEP...). By instead letting :class:`ndarray` return +`NotImplemented` if any argument defines ``__array_ufunc__``, we provide +the option for other classes to have :class:`ndarray` at the bottom of +the type hierarchy. .. admonition:: Example @@ -255,14 +249,14 @@ convert unknown objects to ndarrays. ndarray -> B; } - The ``__array_ufunc__`` of type A can handle ndarrays, B can handle ndarray and D, - and C can handle A and B but not ndarrays or D. The resulting graph is a DAG, - and defines a type casting hierarchy, with relations ``C > A > - ndarray``, ``C > B > ndarray``, ``C > B > D``. The type B is incompatible - relative to A and vice versa, and A and ndarray are incompatible relative to D. - Ufunc expressions involving these classes produce results of the highest type - involved or raise a TypeError. - + The ``__array_ufunc__`` of type A can handle ndarrays, B can handle + ndarray and D, and C can handle A and B but not ndarrays or D. The + result is a directed acyclic graph, and defines a type casting + hierarchy, with relations ``C > A > ndarray``, ``C > B > ndarray``, + ``C > B > D``. The type B is incompatible relative to A and vice + versa, and A and ndarray are incompatible relative to D. Ufunc + expressions involving these classes should produce results of the + highest type involved or raise a :exc:`TypeError`. Subclass hierarchies -------------------- @@ -273,7 +267,8 @@ type casting hierarchy. The recommendation is that an `NotImplemented` unless the inputs are instances of the same class or superclasses. This guarantees that in the type casting hierarchy, superclasses are below, subclasses above, and other classes are -incompatible. Exceptions to this need to check they respect the +incompatible (sadly, the terminology for graphs and classes has reversed +vertical sense). Exceptions to this need to check they respect the implicit type casting hierarchy. Subclasses can be easily constructed if methods consistently use @@ -282,55 +277,82 @@ this, :class:`ndarray` has its own ``__array_ufunc__`` method, equivalent to:: def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - # Handle items of type(self), superclasses, and items - # without __array_ufunc__. Bail out in other cases. - for item in inputs: - if isinstance(self, type(item)) or not hasattr(item, '__array_ufunc__'): - pass - else: + # Cannot handle items that have __array_ufunc__ (other than our own). + outputs = kwargs.get('out', ()) + for item in inputs + outputs): + if (hasattr(item, '__array_ufunc__') and + type(item).__array_ufunc__ is not ndarray.__array_ufunc__): return NotImplemented - # Perform ufunc on the underlying ndarrays (no __array_ufunc__ dispatch) - items = [np.asarray(item) if isinstance(item, np.ndarray) else item - for item in inputs] - result = getattr(ufunc, method)(*items, **kwargs) - - # Cast output to type(self), unless `out` specified - if kwargs['out']: - return result - - if isinstance(result, tuple): - return tuple(x.view(type(self)) for x in result) - else: - return result.view(type(self)) + # If we didn't have to support legacy behaviour (__array_prepare__, + # __array_wrap__, etc.), we might here convert python floats, + # lists, etc, to arrays with + # items = [np.asarray(item) for item in inputs] + # and then start the right iterator for the given method. + # However, we do have to support legacy, so call back into the ufunc. + # Its arguments are now guaranteed not to have __array_ufunc__ + # overrides, and it will do the coercion to array for us. + return getattr(ufunc, method)(*items, **kwargs) Note that, as a special case, the ufunc dispatch mechanism does not call this `ndarray.__array_ufunc__` method, even for `ndarray` subclasses if they have not overridden the default `ndarray` implementation. As a consequence, calling `ndarray.__array_ufunc__` will not result to a -nested ufunc dispatch cycle. Custom implementations of -`__array_ufunc__` should generally avoid nested dispatch cycles. +nested ufunc dispatch cycle. + +The use of :func:`super` should be particularly useful for subclasses of +:class:`ndarray` that only add an attribute like a unit. In their +`__array_ufunc__` implementation, such classes can do possible +adjustment of the arguments relevant to their own class, and pass on to +the superclass implementation using :func:`super` until the ufunc is +actually done, and then do possible adjustments of the outputs. + +In general, custom implementations of `__array_ufunc__` should avoid +nested dispatch cycles. However, for some subclasses, it may be better +to use ``getattr(ufunc, method)(*items, **kwargs)``. For instance, for a +class like :class:`MaskedArray`, which only cares that whatever +it contains is an :class:`ndarray` subclass, a reimplementation with +``__array_ufunc__`` is probably more easily done by directly applying +the ufunc to its data, and then adjusting the mask. + +As a specific example, consider a quantity and a masked array class +which both override ``__array_ufunc__``, with specific instances ``q`` +and ``ma``. For those, an expression like ``q * ma`` will be translated +to ``np.multiply(q, ma)``. The ufunc will first dispatch to +``q.__array_ufunc__``, which returns :obj:`NotImplemented` (since the +quantity class turns itself into an array and calls :func:`super`, which +passes on to ``ndarray.__array_ufunc__``, which sees the override on +``ma``). Next, ``ma.__array_ufunc__`` gets a chance. It does not know +quantity, and if it were to return :obj:`NotImplemented` as well, an +:exc:`TypeError` would result. But it can also try to evaluate using its +contents ``a = ma.data``, i.e., use ``getattr(ufunc, method)`` to +evaluate ``np.multiply(q, a)``. This again will pass to +``q.__array_ufunc__``, but this time, since ``a`` is a regular array, +it will return a result that is also a quantity. Since this is a +subclass of :class:`ndarray`, ``ma.__array_ufunc__`` can turn this into +a masked array and thus return a result (obviously, if it was not a +array subclass, it could still return :obj:`NotImplemented`). + +Note that in the context of the type hierarchy discussed above this is a +somewhat tricky example, since :class:`MaskedArray` has a strange +position: it is above all subclasses of :class:`ndarray`, in that it can +cast them to its own type, but it does not itself know how to interact +with them in ufuncs. -This should be particularly useful for subclasses of :class:`ndarray`, -which only add an attribute like a unit or mask to a regular -:class:`ndarray`. In their `__array_ufunc__` implementation, such -classes can do possible adjustment of the arguments relevant to their -own class, and pass on to superclass implementation using :func:`super` -until the ufunc is actually done, and then do possible adjustments of -the outputs. Turning Ufuncs off ------------------ -For some classes, Ufuncs make no sense, and, like for other special -methods [8]_, one can indicate Ufuncs are not available by setting -``__array_ufunc__`` to :obj:`None`. Inside a Ufunc, this is -equivalent to unconditionally returning :obj:`NotImplemented`, and thus -will lead to a :exc:`TypeError` (unless another operand implements -``__array_ufunc__`` and specifically knows how to deal with the class). +For some classes, Ufuncs make no sense, and, like for some other special +methods such as ``__hash__`` and ``__iter__`` [8]_, one can indicate +Ufuncs are not available by setting ``__array_ufunc__`` to :obj:`None`. +Inside a Ufunc, this is equivalent to unconditionally returning +:obj:`NotImplemented`, and thus will lead to a :exc:`TypeError` (unless +another operand implements ``__array_ufunc__`` and specifically knows +how to deal with the class). -In the type casting hierarchy, this makes the type incompatible relative -to `ndarray`. +In the type casting hierarchy, this makes it explicit that the type is +incompatible relative to :class:`ndarray`. .. [7] https://rhettinger.wordpress.com/2011/05/26/super-considered-super/ @@ -348,8 +370,9 @@ They have indirect interactions, however, because NumPy's most numerical classes, the easiest way to override binary operations is thus to define ``__array_ufunc__`` and override the corresponding Ufunc. The class can then, like :class:`ndarray` itself, define the -binary operators in terms of Ufuncs. Here, one has to take some care. -E.g., the simplest implementation would be:: +binary operators in terms of Ufuncs. Here, one has to take some care to +ensure that one allows for other classes to indicate they are not +compatible, i.e., implementations should be something like:: class ArrayLike(object): ... @@ -357,11 +380,35 @@ E.g., the simplest implementation would be:: ... return result + # Option 1: call ufunc directly + def __mul__(self, other): + if getattr(other, '__array_ufunc__', False) is None: + return NotImplemented + return np.multiply(self, other) + + def __rmul__(self, other): + return np.multiply(other, self) + + def __imul__(self, other): + return np.multiply(self, other, out=self) + + # Option 2: call into one's own __array_ufunc__ def __mul__(self, other): return self.__array_ufunc__(np.multiply, '__call__', self, other) -Suppose now, however, that ``other`` is class that does not know how to -deal with arrays and ufuncs, but does know how to do multiplication:: + def __rmul__(self, other): + return self.__array_ufunc__(np.multiply, '__call__', other, self) + + def __imul__(self, other): + result = self.__array_ufunc__(np.multiply, '__call__', self, other, + out=self) + if result is NotImplemented: + raise TypeError(...) + +To see why some care is necessary, consider another class ``other`` that +does not know how to deal with arrays and ufuncs, and thus has set +``__array_ufunc__`` to :obj:`None`, but does know how to do +multiplication:: class MyObject(object): __array_ufunc__ = None @@ -374,66 +421,77 @@ deal with arrays and ufuncs, but does know how to do multiplication:: def __rmul__(self, other): return MyObject(4321) -In this case, standard Python override rules combined with the above -discussion would imply:: +For either option above, we get the expected result:: mine = MyObject(0) arr = ArrayLike([0]) - mine * arr # == MyObject(1234) OK - arr * mine # TypeError surprising - -XXX: but it doesn't raise a TypeError, because `__mul__` calls -directly `__array_ufunc__`, which sees the `__array_ufunc__ == None`, and -bails out with `NotImplemented`? - -The reason why this would occur is: because ``MyObject`` is not an -``ArrayLike`` subclass, Python resolves the expression ``arr * mine`` by -calling first ``arr.__mul__``. In the above implementation, this would -just call the Ufunc, which would see that ``mine.__array_ufunc__`` is -:obj:`None` and raise a :exc:`TypeError`. (Note that if ``MyObject`` -is a subclass of :class:`ndarray`, Python calls ``mine.__rmul__`` first.) - -So, a better implementation of the binary operators would check whether -the other class can be dealt with in ``__array_ufunc__`` and, if not, -return :obj:`NotImplemented`:: - - class ArrayLike(object): - ... - def __mul__(self, other): - if getattr(other, '__array_ufunc__', False) is None: - return NotImplemented - return self.__array_ufunc__(np.multiply, '__call__', self, other) - - arr = ArrayLike([0]) - - arr * mine # == 4321 OK - -Indeed, after long discussion about whether it might make more sense to -ask classes like ``ArrayLike`` to implement a full ``__array_ufunc__`` -[6]_, the same design as the above was agreed on for :class:`ndarray` -itself. + mine * arr # -> MyObject(1234) + mine *= arr # -> MyObject(1234) + arr * mine # -> MyObject(4321) + arr *= mine # -> TypeError + +Here, in the first and second example, ``mine.__mul__(arr)`` gets called +and the result arrives immediately. In the third example, first +``arr.__mul__(mine)`` is called. In option (1), the check on +``mine.__array_ufunc__ is None`` will succeed and thus +:obj:`NotImplemented` is returned, which causes ``mine.__rmul__(arg)`` +to be executed. In option (2), it is presumably inside +``arr.__array_ufunc__`` that it becomes clear that the other argument +cannot be dealt with, and again :obj:`NotImplemented` is returned, +causing control to pass to ``mine.__rmul__``. + +For the fourth example, with the in-place operators, we have here +followed :class:`ndarray` and ensure we never return +:obj:`NotImplemented`, but rather raise a :exc:`TypeError`. In +option (1) this happens indirectly: we pass to ``np.multiply``, which +calls ``arr.__array_ufunc__``. This, however, will not know what to do +with ``mine`` and will thus return :obj:`NotImplemented`. Then, the +ufunc turns to ``mine.__array_ufunc__``. But this is :obj:`None`, +equivalent to returning :obj:`NotImplemented`, so a :exc:`TypeError` is +raised. In option (2), we pass directly to ``arr.__array_ufunc__``, +which will return :obj:`NotImplemted`, which we catch. + +.. note :: the reason for not allowing in-place operations to return + :obj:`NotImplemented` is that these cannot generically be replaced by + a simple reverse operation: most array operations assume the contents + of the instance are changed in-place, and do not expect a new + instance. Also, what would ``ndarr[:] *= mine`` imply? Assuming it + means ``ndarr[:] = ndarr[:] * mine``, as python does by default if + the ``ndarr.__imul__`` were to return :obj:`NotImplemented`, is + likely to be wrong. + +Now consider what would happen if we had not added checks. For option +(1), the relevant case is if we had not checked whether +``__array_func__`` was set to :obj:`None`. In the third example, +``arr.__mul__(mine)`` is called, and without the check, this would go to +``np.multiply(arr, mine)``. This tries ``arr.__array_ufunc__``, which +returns :obj:`NotImplemented` and sees that ``mine.__array_ufunc__ is +None``, so a :exc:`TypeError` is raised. + +For option (2), the relevant example is the fourth, with ``arr *= +mine``: if we had let the :obj:`NotImplemented` pass, python would have +replaced this with ``arr = mine.__rmul__(arr)``, which is not wanted. + +Finally, we note that we had extensive discussion about whether it might +make more sense to ask classes like ``MyObject`` to implement a full +``__array_ufunc__`` [6]_. In the end, allowing classes to opt out was +preferred, and the above reasoning led us to agree on a similar +implementation for :class:`ndarray` itself. To help implement array-like +classes, we will provide a mixin class that provides overrides for all +binary operators. -.. note:: The above holds for regular operators. For in-place - operators, :class:`ndarray` never returns - :obj:`NotImplemented`, i.e., ``ndarr *= mine`` would always - lead to a :exc:`TypeError`. This is because for arrays - in-place operations cannot generically be replaced by a simple - reverse operation. For instance, sticking to the above - example, what would ``ndarr[:] *= mine`` imply? Assuming it - means ``ndarr[:] = ndarr[:] * mine``, as python does by - default, is likely to be wrong. Extension to other numpy functions ---------------------------------- The ``__array_ufunc__`` method is used to override :func:`~numpy.dot` and :func:`~numpy.matmul` as well, since while these functions are not -(yet) implemented as (generalized) Ufuncs, they are very similar. For -other functions, such as :func:`~numpy.median`, :func:`~numpy.min`, -etc., implementations as (generalized) Ufuncs may well be possible and -logical as well, in which case it will become possible to override these -as well. +Ufuncs, they are very similar. Indeed, :func:`~numpy.matmul` may well +be implemented as a (generalized) Ufunc in the future, as may happen +with some other functions, such as :func:`~numpy.median`, +:func:`~numpy.min`, etc. (in which it will become possible to override +these as well). Demo ==== From 5f9252cc299149d213aa95dfc89b1131060a1289 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Mon, 3 Apr 2017 21:01:19 -0400 Subject: [PATCH 23/43] DOC: implement many smaller and bigger changes suggested in review. --- doc/neps/ufunc-overrides.rst | 136 ++++++++++++++++++++++++-------- doc/source/reference/ufuncs.rst | 2 + numpy/core/tests/test_umath.py | 8 +- numpy/doc/subclassing.py | 33 ++++---- 4 files changed, 126 insertions(+), 53 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 2801802a9638..4626c27fd1d7 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -137,18 +137,21 @@ signature is:: Here: - *ufunc* is the ufunc object that was called. -- *method* is a string indicating which Ufunc method was called - (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, - ``"accumulate"``, ``"outer"``, ``"inner"``). +- *method* is a string indicating how the Ufunc was called, either + ``"__call__"`` to indicate it was called directly, or one of its + :ref:`methods`: ``"reduce"``, ``"accumulate"``, + ``"reduceat"``, ``"outer"``, or ``"at"``. - *inputs* is a tuple of the input arguments to the ``ufunc`` -- *kwargs* are the keyword arguments passed to the function. The ``out`` - arguments are always contained as a tuple in *kwargs*. +- *kwargs* contains any optional or keyword arguments passed to the + function. This includes any ``out`` arguments, which are always + contained in a tuple. -Hence, the arguments are normalized: only the input data (``inputs``) -are passed on as positional arguments, all the others are passed on as a -dict of keyword arguments (``kwargs``). In particular, if there are -output arguments, positional are otherwise, they are passed on as a -tuple in the ``out`` keyword argument. +Hence, the arguments are normalized: only the required input arguments +(``inputs``) are passed on as positional arguments, all the others are +passed on as a dict of keyword arguments (``kwargs``). In particular, if +there are output arguments, positional are otherwise, that are not +:obj:`None`, they are passed on as a tuple in the ``out`` keyword +argument. The function dispatch proceeds as follows: @@ -191,8 +194,9 @@ but direct A->C not). Moreover, one should make sure the implementations of ``__array_ufunc__``, which implicitly define the type casting hierarchy, don't contradict this. -It is useful to think of the typecasting hierarchy as a graph (see -example below) in which, for any given class A, all other classes that +It is useful to think of the typecasting hierarchy as a graph (see below +for example graphs that work and that fail because of cyclic +dependencies) in which, for any given class A, all other classes that define ``__array_ufunc__`` must belong to exactly one of three groups (making this an directed acyclic graph): @@ -206,7 +210,7 @@ define ``__array_ufunc__`` must belong to exactly one of three groups in ufuncs. - *Incompatible*: neither above nor below A; types for which no - (indirect) upcasting is possible. + (indirect) upcasting is possible. Neither can handle the other. Given this grouping, to ensure that expressions involving ufuncs either raise a :exc:`TypeError`, or have a result type that is independent of @@ -219,24 +223,30 @@ type A should: - Return :obj:`NotImplemented` if any argument has a type that is above A or with which it is incompatible. +With the above, one can convert relations between types to edges in a +`graph`_ by defining "can +handle" as follows: if for instances ``a`` and ``b`` of types A and B, +``a.__array_ufunc__(..., b, ...)`` returns a result other than +:obj:`NotImplemented` (and does not raise an error), then a can handle +b and B->A is an edge of the graph. + Note that there are, as always, exceptions. For instance, for a quantity class, the results of most ufuncs should be quantities, but this is not the case for comparison operators. For those, a quantity class would return a plain array. -Note also that the legacy behaviour of numpy ufunc (legacy) behavior is -to try to convert unknown objects to :class:`ndarray` via -:func:`np.asarray`. This is equivalent to placing :class:`ndarray` at -the very top of the graph, and is thus a consistent type -hierarchy (although one that causes the problems that motivate -this NEP...). By instead letting :class:`ndarray` return -`NotImplemented` if any argument defines ``__array_ufunc__``, we provide -the option for other classes to have :class:`ndarray` at the bottom of -the type hierarchy. +Note also that the legacy behaviour of numpy ufunc is to try to convert +unknown objects to :class:`ndarray` via :func:`np.asarray`. This is +equivalent to placing :class:`ndarray` at the very top of the graph, and +is thus a consistent type hierarchy (although one that causes the +problems that motivate this NEP...). By instead letting +:class:`ndarray` return `NotImplemented` if any argument defines +``__array_ufunc__``, we provide the option for other classes to have +:class:`ndarray` at the bottom of the type hierarchy. .. admonition:: Example - Type casting hierarchy + Type casting hierarchy. .. graphviz:: @@ -258,6 +268,43 @@ the type hierarchy. expressions involving these classes should produce results of the highest type involved or raise a :exc:`TypeError`. +.. admonition:: Example + + One-cycle in the ``__array_ufunc__`` graph. + + .. graphviz:: + + digraph array_ufuncs { + rankdir=BT; + A -> B; + B -> A; + } + + + In this case, the ``__array_ufunc__`` relations have a cycle of length 1, + and a type casting hierarchy does not exist. Binary operations are not + commutative: ``type(a + b) is A`` but ``type(b + a) is B``. + +.. admonition:: Example + + Longer cycle in the ``__array_ufunc__`` graph. + + .. graphviz:: + + digraph array_ufuncs { + rankdir=BT; + A -> B; + B -> C; + C -> A; + } + + + In this case, the ``__array_ufunc__`` relations have a longer cycle, and a + type casting hierarchy does not exist. Binary operations are still + commutative, but type transitivity is lost: ``type(a + (b + c)) is A`` but + ``type((a + b) + c) is C``. + + Subclass hierarchies -------------------- @@ -312,23 +359,42 @@ nested dispatch cycles. However, for some subclasses, it may be better to use ``getattr(ufunc, method)(*items, **kwargs)``. For instance, for a class like :class:`MaskedArray`, which only cares that whatever it contains is an :class:`ndarray` subclass, a reimplementation with -``__array_ufunc__`` is probably more easily done by directly applying -the ufunc to its data, and then adjusting the mask. +``__array_ufunc__`` may well be more easily done by directly applying +the ufunc to its data, and then adjusting the mask. Indeed, one can +think of this as part of the class determining whether it can handle the +other argument (i.e., where in the type hierarchy it sits). In this +case, one should return :obj:`NotImplemented` if the trial fails. So, +the implementation would be something like:: + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + # for simplicity, outputs are ignored here. + unmasked_items = [item.data if isinstance(item, MaskedArray) + else item] + try: + unmasked_result = getattr(ufunc, method)(*unmasked_items, **kwargs) + except TypeError: + return NotImplemented + # for simplicity, ignore that unmasked_result could be a tuple + # or a scalar. + if not isinstance(unmasked_result, np.ndarray): + return NotImplemented + # now combine masks and view as MaskedArray instance + ... As a specific example, consider a quantity and a masked array class which both override ``__array_ufunc__``, with specific instances ``q`` -and ``ma``. For those, an expression like ``q * ma`` will be translated -to ``np.multiply(q, ma)``. The ufunc will first dispatch to +and ``ma``, where the latter contains a regular array. Executing +``np.multiply(q, ma)``, the ufunc will first dispatch to ``q.__array_ufunc__``, which returns :obj:`NotImplemented` (since the quantity class turns itself into an array and calls :func:`super`, which passes on to ``ndarray.__array_ufunc__``, which sees the override on ``ma``). Next, ``ma.__array_ufunc__`` gets a chance. It does not know -quantity, and if it were to return :obj:`NotImplemented` as well, an -:exc:`TypeError` would result. But it can also try to evaluate using its -contents ``a = ma.data``, i.e., use ``getattr(ufunc, method)`` to -evaluate ``np.multiply(q, a)``. This again will pass to -``q.__array_ufunc__``, but this time, since ``a`` is a regular array, -it will return a result that is also a quantity. Since this is a +quantity, and if it were to just return :obj:`NotImplemented` as well, +an :exc:`TypeError` would result. But in our sample implementation, it +uses ``getattr(ufunc, method)`` to, effectively, evaluate +``np.multiply(q, ma.data)``. This again will pass to +``q.__array_ufunc__``, but this time, since ``ma.data`` is a regular +array, it will return a result that is also a quantity. Since this is a subclass of :class:`ndarray`, ``ma.__array_ufunc__`` can turn this into a masked array and thus return a result (obviously, if it was not a array subclass, it could still return :obj:`NotImplemented`). @@ -390,7 +456,7 @@ compatible, i.e., implementations should be something like:: return np.multiply(other, self) def __imul__(self, other): - return np.multiply(self, other, out=self) + return np.multiply(self, other, out=(self,)) # Option 2: call into one's own __array_ufunc__ def __mul__(self, other): @@ -401,7 +467,7 @@ compatible, i.e., implementations should be something like:: def __imul__(self, other): result = self.__array_ufunc__(np.multiply, '__call__', self, other, - out=self) + out=(self,)) if result is NotImplemented: raise TypeError(...) diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 4dd1b3e1823a..bcd3d5f0aaa2 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -416,6 +416,8 @@ possess. None of the attributes can be set. ufunc.types ufunc.identity +.. _ufuncs.methods: + Methods ------- diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index c7d7781be79a..7160d24b2fd1 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1868,9 +1868,11 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out_args.append(output) kwargs['out'] = tuple(out_args) - info = {key: no for (key, no) in (('inputs', in_no), - ('outputs', out_no)) - if no != []} + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no results = super(A, self).__array_ufunc__(ufunc, method, *args, **kwargs) diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index c42d5e330254..7877432b54b3 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -431,20 +431,21 @@ def __array_finalize__(self, obj): def __array_ufunc__(ufunc, method, *inputs, **kwargs): - - *ufunc* is the ufunc object that was called. - - *method* is a string indicating which Ufunc method was called - (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, - ``"accumulate"``, ``"outer"``, ``"inner"``). - - *inputs* is a tuple of the input arguments to the ``ufunc``. - - *kwargs* is a dictionary containing the optional input arguments - of the ufunc. If given, any ``out`` arguments, both positional - and keyword, are passed as a :obj:`tuple` in *kwargs*. + - *ufunc* is the ufunc object that was called. + - *method* is a string indicating how the Ufunc was called, either + ``"__call__"`` to indicate it was called directly, or one of its + :ref:`methods`: ``"reduce"``, ``"accumulate"``, + ``"reduceat"``, ``"outer"``, or ``"at"``. + - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *kwargs* contains any optional or keyword arguments passed to the + function. This includes any ``out`` arguments, which are always + contained in a tuple. A typical implementation would convert any inputs or ouputs that are instances of one's own class, pass everything on to a superclass using ``super()``, and finally return the results after possible back-conversion. An example, taken from the test case -``test_ufunc_override_with_super`` in ``core/tests/test_umath.pu``, is the +``test_ufunc_override_with_super`` in ``core/tests/test_umath.py``, is the following. .. testcode:: @@ -462,9 +463,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): else: args.append(input_) - outputs = kwargs.pop('out', []) + outputs = kwargs.pop('out', None) out_no = [] - if outputs: + if outputs is not None: out_args = [] for j, output in enumerate(outputs): if isinstance(output, A): @@ -474,9 +475,11 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out_args.append(output) kwargs['out'] = tuple(out_args) - info = {key: no for (key, no) in (('inputs', in_no), - ('outputs', out_no)) - if no != []} + info = {} + if in_no: + info['inputs'] = in_no + if out_no: + info['outputs'] = out_no results = super(A, self).__array_ufunc__(ufunc, method, *args, **kwargs) @@ -488,7 +491,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return results results = (results,) - if outputs == []: + if outputs is None: outputs = [None] * len(results) results = tuple(result.view(A) if output is None else output for result, output in zip(results, outputs)) From 8cc2f71d3b2a37506918dc9a4f9d078c567ca7be Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Mon, 3 Apr 2017 21:34:33 -0400 Subject: [PATCH 24/43] BUG,MAINT: ensure out=None is never passed on to __array_ufunc__. As part of this, it turns out the number of output arguments nout was needed, so that was passed in. Also, testing showed that arguments passed in both as positional and keyword arguments were not caught, so a duplication check was added. --- numpy/core/src/multiarray/methods.c | 2 +- numpy/core/src/multiarray/multiarraymodule.c | 4 +- numpy/core/src/private/ufunc_override.c | 299 +++++++++++-------- numpy/core/src/private/ufunc_override.h | 2 +- numpy/core/src/umath/ufunc_object.c | 27 +- numpy/core/tests/test_multiarray.py | 8 +- numpy/core/tests/test_umath.py | 76 ++++- 7 files changed, 279 insertions(+), 139 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 6cfd05cd6634..6762279dffea 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -2106,7 +2106,7 @@ array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", - newargs, NULL, &override, 2); + newargs, NULL, &override, 2, 1); Py_DECREF(newargs); if (errval) { diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 27e1e2af29f7..9d259cb03d62 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2196,7 +2196,7 @@ array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) } errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwds, - &override, 2); + &override, 2, 1); if (errval) { return NULL; } @@ -2370,7 +2370,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) } errval = PyUFunc_CheckOverride((PyUFuncObject*)matmul, "__call__", - args, kwds, &override, 2); + args, kwds, &override, 2, 1); if (errval) { return NULL; } diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index 1db4e54b979a..14cf9b39b057 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -8,28 +8,55 @@ #include "ufunc_override.h" -static void + +static int normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds, - int nin) + PyObject **normal_args, PyObject **normal_kwds, + int nin, int nout) { /* ufunc.__call__(*args, **kwds) */ int i; int not_all_none; int nargs = PyTuple_GET_SIZE(args); - PyObject *obj = PyDict_GetItemString(*normal_kwds, "sig"); + PyObject *obj; + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "required input argument (pos %d) not found", nin); + return -1; + } + if (nargs > nin+nout) { + PyErr_Format(PyExc_TypeError, + "ufunc takes at most %d arguments (%d given)", + nin+nout, nargs); + return -1; + } /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + obj = PyDict_GetItemString(*normal_kwds, "sig"); if (obj != NULL) { + if (PyDict_GetItemString(*normal_kwds, "signature")) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + return -1; + } Py_INCREF(obj); PyDict_SetItemString(*normal_kwds, "signature", obj); PyDict_DelItemString(*normal_kwds, "sig"); } *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } /* If we have more args than nin, they must be the output variables.*/ if (nargs > nin) { + if(PyDict_GetItemString(*normal_kwds, "out")) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('out') and position (%d)", + nin); + return -1; + } for (i=nin; i < nargs; i++) { not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); if (not_all_none) { @@ -37,119 +64,116 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, } } if (not_all_none) { - obj = PyTuple_GetSlice(args, nin, nargs); + if (nargs - nin == nout) + { + obj = PyTuple_GetSlice(args, nin, nargs); + } + else { + PyObject *item; + + obj = PyTuple_New(nout); + if (obj == NULL) { + return -1; + } + for (i = 0; i < nout; i++) { + if (i + nin < nargs) { + item = PyTuple_GET_ITEM(args, nin+i); + } + else { + item = Py_None; + } + Py_INCREF(item); + PyTuple_SET_ITEM(obj, i, item); + } + } PyDict_SetItemString(*normal_kwds, "out", obj); Py_DECREF(obj); } } + return 0; } -static void -normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) +static int +normalize_reduce_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.reduce(a[, axis, dtype, out, keepdims]) */ + /* + * ufunc.reduce(a[, axis, dtype, out, keepdims]) + * ufunc.accumulate(a[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ int nargs = PyTuple_GET_SIZE(args); int i; PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; *normal_args = PyTuple_GetSlice(args, 0, 1); - for (i = 1; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (obj == Py_None) { - continue; - } - if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else if (i == 3) { - /* out */ - obj = PyTuple_GetSlice(args, 3, 4); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } - else { - /* keepdims */ - PyDict_SetItemString(*normal_kwds, "keepdims", obj); - } + if (*normal_args == NULL) { + return -1; } - return; -} - -static void -normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.accumulate(a[, axis, dtype, out]) */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - *normal_args = PyTuple_GetSlice(args, 0, 1); for (i = 1; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (obj == Py_None) { - continue; - } - if (i == 1) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); - } - else if (i == 2) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; } - else { - /* out */ - obj = PyTuple_GetSlice(args, 3, 4); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } } } - return; + return 0; } -static void +static int normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.reduceat(a, indicies[, axis, dtype, out]) */ + /* + * ufunc.reduceat(a, indicies[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ int i; int nargs = PyTuple_GET_SIZE(args); PyObject *obj; + static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; /* a and indicies */ *normal_args = PyTuple_GetSlice(args, 0, 2); + if (*normal_args == NULL) { + return -1; + } for (i = 2; i < nargs; i++) { - obj = PyTuple_GET_ITEM(args, i); - if (obj == Py_None) { - continue; - } - if (i == 2) { - /* axis */ - PyDict_SetItemString(*normal_kwds, "axis", obj); + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; } - else if (i == 3) { - /* dtype */ - PyDict_SetItemString(*normal_kwds, "dtype", obj); - } - else { - /* out */ - obj = PyTuple_GetSlice(args, 4, 5); - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 4) { + obj = PyTuple_GetSlice(args, 4, 5); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 4) { + Py_DECREF(obj); + } } } - return; + return 0; } -static void +static int normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { @@ -158,10 +182,10 @@ normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, * This has no kwds so we don't need to do any kwd stuff. */ *normal_args = PyTuple_GetSlice(args, 0, 2); - return; + return (*normal_args == NULL); } -static void +static int normalize_at_args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { @@ -169,7 +193,7 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, int nargs = PyTuple_GET_SIZE(args); *normal_args = PyTuple_GetSlice(args, 0, nargs); - return; + return (*normal_args == NULL); } /* @@ -236,14 +260,14 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, * Check inputs */ if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_TypeError, "Internal Numpy error: call to PyUFunc_CheckOverride " "with non-tuple"); goto fail; } nargs = PyTuple_GET_SIZE(args); if (nargs > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_TypeError, "Internal Numpy error: too many arguments in call " "to PyUFunc_CheckOverride"); goto fail; @@ -301,21 +325,28 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, * * Returns 0 on success and 1 on exception. On success, *result contains the * result of the operation, if any. If *result is NULL, there is no override. + * + * TODO: the ufunc really should always be a ufunc, so that we can rely on + * using, e.g., ufunc->nin, ufunc->nout, etc. Right now, we cannot, since we + * also use this function to override np.dot and np.matmul. This should be + * fixed. */ NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, PyObject **result, - int nin) + int nin, int nout) { int i; int j; + int status; int noa; PyObject *with_override[NPY_MAXARGS]; PyObject *obj; PyObject *other_obj; + PyObject *out; PyObject *method_name = NULL; PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ @@ -340,7 +371,6 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Build new kwds */ if (kwds && PyDict_CheckExact(kwds)) { - PyObject *out; /* ensure out is always a tuple */ normal_kwds = PyDict_Copy(kwds); @@ -348,8 +378,13 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, if (out != NULL) { if (PyTuple_Check(out)) { int all_none = 1; - int i; + if (PyTuple_GET_SIZE(out) != nout) { + PyErr_Format(PyExc_TypeError, + "The 'out' tuple must have exactly " + "%d entries: one per ufunc output", nout); + goto fail; + } for (i = 0; i < PyTuple_GET_SIZE(out); i++) { all_none = (PyTuple_GET_ITEM(out, i) == Py_None); if (!all_none) { @@ -360,19 +395,45 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyDict_DelItemString(normal_kwds, "out"); } } - else if (out != Py_None) { - /* not already a tuple and not None */ - PyObject *out_tuple = PyTuple_New(1); - - if (out_tuple == NULL) { + else { + /* not a tuple */ + if (nout > 1 && DEPRECATE("passing a single argument to the " + "'out' keyword argument of a " + "ufunc with\n" + "more than one output will " + "result in an error in the " + "future") < 0) { + /* + * If the deprecation is removed, also remove the loop + * below setting tuple items to None (but keep this future + * error message.) + */ + PyErr_SetString(PyExc_TypeError, + "'out' must be a tuple of arguments"); goto fail; } - /* out was borrowed ref; make it permanent */ - Py_INCREF(out); - /* steals reference */ - PyTuple_SET_ITEM(out_tuple, 0, out); - PyDict_SetItemString(normal_kwds, "out", out_tuple); - Py_DECREF(out_tuple); + if (out != Py_None) { + /* not already a tuple and not None */ + PyObject *out_tuple = PyTuple_New(nout); + + if (out_tuple == NULL) { + goto fail; + } + for (i = 1; i < nout; i++) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(out_tuple, i, Py_None); + } + /* out was borrowed ref; make it permanent */ + Py_INCREF(out); + /* steals reference */ + PyTuple_SET_ITEM(out_tuple, 0, out); + PyDict_SetItemString(normal_kwds, "out", out_tuple); + Py_DECREF(out_tuple); + } + else { + /* out=None; remove it */ + PyDict_DelItemString(normal_kwds, "out"); + } } } } @@ -387,35 +448,37 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* ufunc.__call__ */ if (strcmp(method, "__call__") == 0) { - normalize___call___args(ufunc, args, &normal_args, &normal_kwds, nin); - } - - /* ufunc.reduce */ - else if (strcmp(method, "reduce") == 0) { - normalize_reduce_args(ufunc, args, &normal_args, &normal_kwds); + status = normalize___call___args(ufunc, args, &normal_args, + &normal_kwds, nin, nout); } - - /* ufunc.accumulate */ - else if (strcmp(method, "accumulate") == 0) { - normalize_accumulate_args(ufunc, args, &normal_args, &normal_kwds); + /* ufunc.reduce and ufunc.accumulate */ + else if ((strcmp(method, "reduce") == 0) || + (strcmp(method, "accumulate") == 0)) { + status = normalize_reduce_accumulate_args(ufunc, args, &normal_args, + &normal_kwds); } - /* ufunc.reduceat */ else if (strcmp(method, "reduceat") == 0) { - normalize_reduceat_args(ufunc, args, &normal_args, &normal_kwds); + status = normalize_reduceat_args(ufunc, args, &normal_args, + &normal_kwds); } - /* ufunc.outer */ else if (strcmp(method, "outer") == 0) { - normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); } - /* ufunc.at */ else if (strcmp(method, "at") == 0) { - normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); } - - if (normal_args == NULL) { + /* unknown method */ + else { + PyErr_Format(PyExc_TypeError, + "Internal Numpy error: unknown ufunc method '%s' in call " + "to PyUFunc_CheckOverride", method); + status = -1; + } + if (status != 0) { + Py_XDECREF(normal_args); goto fail; } diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index db5e84fd5871..68dd0221d0b0 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -12,5 +12,5 @@ NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, PyObject **result, - int nin); + int nin, int nout); #endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 22a73e6ba919..605d59e617a5 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -4371,7 +4371,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) } errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override, - ufunc->nin); + ufunc->nin, ufunc->nout); if (errval) { return NULL; } @@ -5087,8 +5087,9 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return NULL; } - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override, 0); + /* Note: `nin` and `nout` are not used in the normalization */ + errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override, + ufunc->nin, ufunc->nout); if (errval) { return NULL; } @@ -5165,8 +5166,9 @@ ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, 0); + /* `nin` and `nout`, the last two arguments, are not actually used */ + errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, + 1, ufunc->nout); if (errval) { return NULL; } @@ -5182,8 +5184,9 @@ ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, 0); + /* `nin` and `nout`, the last two arguments, are not actually used */ + errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, + 1, ufunc->nout); if (errval) { return NULL; } @@ -5199,8 +5202,9 @@ ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, 0); + /* `nin` and `nout`, the last two arguments, are not actually used */ + errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, + ufunc->nin, ufunc->nout); if (errval) { return NULL; } @@ -5264,8 +5268,9 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) char * err_msg = NULL; NPY_BEGIN_THREADS_DEF; - /* `nin`, the last arg, is unused. So we put 0. */ - errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override, 0); + /* `nin` and `nout`, the last two arguments, are not actually used */ + errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override, + ufunc->nin + 1, 0); if (errval) { return NULL; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 5da66da0c36a..f03b5d32398b 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -3115,13 +3115,17 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kw): assert_equal(np.modf(dummy, a), (0,)) assert_equal(np.modf(dummy, None, a), (1,)) assert_equal(np.modf(dummy, dummy, a), (1,)) - assert_equal(np.modf(dummy, out=a), (0,)) - assert_equal(np.modf(dummy, out=(a,)), (0,)) assert_equal(np.modf(dummy, out=(a, None)), (0,)) assert_equal(np.modf(dummy, out=(a, dummy)), (0,)) assert_equal(np.modf(dummy, out=(None, a)), (1,)) assert_equal(np.modf(dummy, out=(dummy, a)), (1,)) assert_equal(np.modf(a, out=(dummy, a)), 0) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_equal(np.modf(dummy, out=a), (0,)) + assert_(w[0].category is DeprecationWarning) + assert_raises(TypeError, np.modf, dummy, out=(a,)) + # 2 inputs, 1 output assert_equal(np.add(a, dummy), 0) assert_equal(np.add(dummy, a), 1) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 7160d24b2fd1..62a2e5c005d1 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1720,6 +1720,17 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): 'keepdims': 'keep0', 'axis': 'axis0'}) + # reduce, output equal to None removed. + res = np.multiply.reduce(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.reduce(a, out=(None,)) + assert_equal(res[4], {}) + + # reduce, wrong args + assert_raises(TypeError, np.multiply.reduce, a, out=()) + assert_raises(TypeError, np.multiply.reduce, a, out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, 'axis0', axis='axis0') + # accumulate, pos args res = np.multiply.accumulate(a, 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) @@ -1741,6 +1752,19 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): 'out': ('out0',), 'axis': 'axis0'}) + # accumulate, output equal to None removed. + res = np.multiply.accumulate(a, out=None) + assert_equal(res[4], {}) + res = np.multiply.accumulate(a, out=(None,)) + assert_equal(res[4], {}) + + # accumulate, wrong args + assert_raises(TypeError, np.multiply.accumulate, a, out=()) + assert_raises(TypeError, np.multiply.accumulate, a, + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.accumulate, a, + 'axis0', axis='axis0') + # reduceat, pos args res = np.multiply.reduceat(a, [4, 2], 'axis0', 'dtype0', 'out0') assert_equal(res[0], a) @@ -1762,6 +1786,19 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): 'out': ('out0',), 'axis': 'axis0'}) + # reduceat, output equal to None removed. + res = np.multiply.reduceat(a, [4, 2], out=None) + assert_equal(res[4], {}) + res = np.multiply.reduceat(a, [4, 2], out=(None,)) + assert_equal(res[4], {}) + + # reduceat, wrong args + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], out=()) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + out=('out0', 'out1')) + assert_raises(TypeError, np.multiply.reduce, a, [4, 2], + 'axis0', axis='axis0') + # outer res = np.multiply.outer(a, 42) assert_equal(res[0], a) @@ -1811,6 +1848,29 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_equal(res7['out'][0], 'out0') assert_equal(res7['out'][1], 'out1') + # While we're at it, check that default output is never passed on. + assert_(np.sin(a, None) == {}) + assert_(np.sin(a, out=None) == {}) + assert_(np.sin(a, out=(None,)) == {}) + assert_(np.modf(a, None) == {}) + assert_(np.modf(a, None, None) == {}) + assert_(np.modf(a, out=(None, None)) == {}) + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', '', DeprecationWarning) + assert_(np.modf(a, out=None) == {}) + assert_(w[0].category is DeprecationWarning) + + # don't give positional and output argument, or too many arguments. + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, np.multiply, a, b, 'one', out='two') + assert_raises(TypeError, np.multiply, a, b, 'one', 'two') + assert_raises(TypeError, np.multiply, a, b, out=('one', 'two')) + assert_raises(TypeError, np.multiply, a, out=()) + assert_raises(TypeError, np.modf, a, 'one', out=('two', 'three')) + assert_raises(TypeError, np.modf, a, 'one', 'two', 'three') + assert_raises(TypeError, np.modf, a, out=('one', 'two', 'three')) + assert_raises(TypeError, np.modf, a, out=('one',)) + def test_ufunc_override_exception(self): class A(object): @@ -1829,20 +1889,28 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return self, ufunc, method, inputs, kwargs a = A() - res = np.core.umath_tests.inner1d(a, a) + inner1d = np.core.umath_tests.inner1d + res = inner1d(a, a) assert_equal(res[0], a) - assert_equal(res[1], np.core.umath_tests.inner1d) + assert_equal(res[1], inner1d) assert_equal(res[2], '__call__') assert_equal(res[3], (a, a)) assert_equal(res[4], {}) - res = np.core.umath_tests.inner1d(1, 1, out=a) + res = inner1d(1, 1, out=a) assert_equal(res[0], a) - assert_equal(res[1], np.core.umath_tests.inner1d) + assert_equal(res[1], inner1d) assert_equal(res[2], '__call__') assert_equal(res[3], (1, 1)) assert_equal(res[4], {'out': (a,)}) + # wrong number of arguments in the tuple is an error too. + assert_raises(TypeError, inner1d, a, out='two') + assert_raises(TypeError, inner1d, a, a, 'one', out='two') + assert_raises(TypeError, inner1d, a, a, 'one', 'two') + assert_raises(TypeError, inner1d, a, a, out=('one', 'two')) + assert_raises(TypeError, inner1d, a, a, out=()) + def test_ufunc_override_with_super(self): class A(np.ndarray): From 856da73546da7adf8511a1d2cf83c23db171f26b Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Wed, 5 Apr 2017 15:57:08 -0400 Subject: [PATCH 25/43] DOC: remove left-over piece discussing binops The text had been changed following the suggestion of @njsmith, but somehow I had forgotten to delete the original. --- doc/source/reference/arrays.classes.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 1ece99af613f..838b5076077f 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -88,17 +88,6 @@ NumPy provides several hooks that classes can customize: unless another class also provides a :func:`__array_ufunc__` method which knows what to do with your class). - The presence of :func:`__array_ufunc__` also influences how binary - and comparison operators are dealt with, such as ``__add__``, - ``__gt__``, etc. If it is not :obj:`None`, the assumption is that - your code can handle such operations via the ufunc mechanism, and - hence forward methods on :class:`ndarray` will call the ufuncs - unconditionally (i.e., even if your class has defined reverse - methods such as ``__radd__``, ``__le__``, etc.). If - ``__array_ufunc__ = None``, however, forward methods on - :class:`ndarray` will unconditionally return :obj:`NotImplemented`, - so that your reverse methods will get called. - The presence of :func:`__array_ufunc__` also influences how :class:`ndarray` handles binary operations like ``arr + obj`` and ``arr < obj`` when ``arr`` is an :class:`ndarray` and ``obj`` is an instance From 2b6c7fd7e40fb9eaee515826f40f6449af191098 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Thu, 6 Apr 2017 10:25:27 -0400 Subject: [PATCH 26/43] REVERT: remove __array_ufunc__ override for np.dot and ndarray.dot. --- doc/neps/ufunc-overrides.rst | 30 ++++++------- doc/source/reference/arrays.classes.rst | 11 ++--- numpy/core/src/multiarray/methods.c | 45 +++++--------------- numpy/core/src/multiarray/multiarraymodule.c | 39 +++++------------ numpy/core/tests/test_multiarray.py | 24 ----------- numpy/core/tests/test_umath.py | 6 +-- 6 files changed, 44 insertions(+), 111 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 4626c27fd1d7..724d9ac115b8 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -551,13 +551,13 @@ binary operators. Extension to other numpy functions ---------------------------------- -The ``__array_ufunc__`` method is used to override :func:`~numpy.dot` -and :func:`~numpy.matmul` as well, since while these functions are not -Ufuncs, they are very similar. Indeed, :func:`~numpy.matmul` may well -be implemented as a (generalized) Ufunc in the future, as may happen -with some other functions, such as :func:`~numpy.median`, -:func:`~numpy.min`, etc. (in which it will become possible to override -these as well). +The ``__array_ufunc__`` method is also used to override +:func:`~numpy.matmul`, since while this function is not a Ufunc, it is +very similar. Indeed, :func:`~numpy.matmul` may well be implemented as +a (generalized) Ufunc in the future, as may happen with some other +functions, such as :func:`~numpy.median`, :func:`~numpy.min`, +:func:`~numpy.argsort`, etc. (in which case it will thus become possible +to override these as well). Demo ==== @@ -576,19 +576,19 @@ proposed in this NEP. Here is a demo highlighting the functionality.:: In [4]: b = B() - In [5]: np.dot(a, b) + In [5]: np.matmul(a, b) Out[5]: 'B' In [6]: np.multiply(a, b) Out[6]: 'B' As a simple example, one could add the following ``__array_ufunc__`` to -SciPy's sparse matrices (just for ``np.dot`` and ``np.multiply`` as +SciPy's sparse matrices (just for ``np.matmul`` and ``np.multiply`` as these are the two most common cases where users would attempt to use sparse matrices with ufuncs):: def __array_ufunc__(self, func, method, pos, inputs, **kwargs): - """Method for compatibility with NumPy's ufuncs and dot + """Method for compatibility with NumPy's ufuncs and matmul functions. """ @@ -599,7 +599,7 @@ sparse matrices with ufuncs):: if func is np.multiply: return self.multiply(*without_self) - elif func is np.dot: + elif func is np.matmul: if pos == 0: return self.__mul__(inputs[1]) if pos == 1: @@ -617,19 +617,19 @@ So we now get the expected behavior when using ufuncs with sparse matrices.:: In [4]: asp = sp.csr_matrix(a); bsp = sp.csr_matrix(b) - In [5]: np.dot(a,b) + In [5]: np.matmul(a,b) Out[5]: array([[2, 4, 8], [2, 4, 8], - [2, 2, 3]]) + [2, 2, 3]]) - In [6]: np.dot(asp,b) + In [6]: np.matmul(asp,b) Out[6]: array([[2, 4, 8], [2, 4, 8], [2, 2, 3]], dtype=int64) - In [7]: np.dot(asp, bsp).A + In [7]: np.matmul(asp, bsp).A Out[7]: array([[2, 4, 8], [2, 4, 8], diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 838b5076077f..10a776e8dfdc 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -74,11 +74,12 @@ NumPy provides several hooks that classes can customize: :exc:`TypeError` is raised. .. note:: In addition to ufuncs, :func:`__array_ufunc__` also - overrides the behavior of :func:`numpy.dot` and :func:`numpy.matmul`. - This even though these are not ufuncs, but they can be thought of as - :ref:`generalized universal functions` - (which are overridden). We intend to extend this behaviour to other - relevant functions. + overrides the behavior of :func:`numpy.matmul`, even though it + is not a ufunc. But it can be thought of as :ref:`generalized + universal functions` (which are + overridden). Indeed, we intend to rewrite :func:`numpy.matmul` + as such, and possibly do the same to other relevant functions, + such as :func:`numpy.median`, :func:`numpy.argsort`, etc. Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 6762279dffea..679e723a6e8d 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -2080,11 +2080,7 @@ array_cumprod(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; - PyObject *a = (PyObject *)self, *b, *o = Py_None; - PyObject *newargs; + PyObject *a = (PyObject *)self, *b, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"b", "out", NULL }; @@ -2093,36 +2089,15 @@ array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - if ((newargs = PyTuple_Pack(3, a, b, o)) == NULL) { - return NULL; - } - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", - newargs, NULL, &override, 2, 1); - Py_DECREF(newargs); - - if (errval) { - return NULL; - } - else if (override) { - return override; - } - - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o != NULL) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, + "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, b, (PyArrayObject *)o); return PyArray_Return(ret); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 9d259cb03d62..4cc2b481c9ce 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2179,41 +2179,22 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args) static PyObject * array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) { - static PyUFuncObject *cached_npy_dot = NULL; - int errval; - PyObject *override = NULL; PyObject *v, *a, *o = NULL; PyArrayObject *ret; char* kwlist[] = {"a", "b", "out", NULL }; - if (cached_npy_dot == NULL) { - PyObject *module = PyImport_ImportModule("numpy.core.multiarray"); - cached_npy_dot = (PyUFuncObject*)PyDict_GetItemString( - PyModule_GetDict(module), "dot"); - - Py_INCREF(cached_npy_dot); - Py_DECREF(module); - } - - errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwds, - &override, 2, 1); - if (errval) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", + kwlist, &a, &v, &o)) { return NULL; } - else if (override) { - return override; - } - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", kwlist, &a, &v, &o)) { - return NULL; - } - if (o == Py_None) { - o = NULL; - } - if (o != NULL && !PyArray_Check(o)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (o) { + if (o == Py_None) { + o = NULL; + } + else if (!PyArray_Check(o)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } ret = (PyArrayObject *)PyArray_MatrixProduct2(a, v, (PyArrayObject *)o); return PyArray_Return(ret); diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index f03b5d32398b..bed437f40375 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2406,30 +2406,6 @@ def test_dot(self): a.dot(b=b, out=c) assert_equal(c, np.dot(a, b)) - def test_dot_override(self): - class B(object): - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - return "B" - - class C(object): - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - return NotImplemented - - class D(object): - __array_ufunc__ = None - - a = np.array([[1]]) - b = B() - c = C() - d = D() - - assert_equal(np.dot(a, b), "B") - assert_equal(a.dot(b), "B") - assert_raises(TypeError, np.dot, c, a) - assert_raises(TypeError, a.dot, c) - assert_raises(TypeError, np.dot, d, a) - assert_raises(TypeError, a.dot, d) - def test_dot_type_mismatch(self): c = 1. A = np.array((1,1), dtype='i,i') diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 62a2e5c005d1..1fef5f50f604 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1577,13 +1577,13 @@ def __array_ufunc__(self, func, method, *inputs, **kwargs): a = A() b = np.matrix([1]) res0 = np.multiply(a, b) - res1 = np.dot(a, b) + res1 = np.matmul(a, b) # self assert_equal(res0[0], a) assert_equal(res1[0], a) assert_equal(res0[1], np.multiply) - assert_equal(res1[1], np.dot) + assert_equal(res1[1], np.matmul) assert_equal(res0[2], '__call__') assert_equal(res1[2], '__call__') assert_equal(res0[3], (a, b)) @@ -1878,7 +1878,7 @@ def __array_ufunc__(self, *a, **kwargs): raise ValueError("oops") a = A() - for func in [np.divide, np.dot]: + for func in [np.divide, np.matmul]: assert_raises(ValueError, func, a, a) def test_gufunc_override(self): From 36e84948a448c74efda008a9629c68e9fbb0a218 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Thu, 6 Apr 2017 12:19:13 -0400 Subject: [PATCH 27/43] REVERT: remove __array_ufunc__ override for np.matmul. --- doc/neps/ufunc-overrides.rst | 29 +++++++------- doc/source/reference/arrays.classes.rst | 15 +++---- numpy/core/src/multiarray/multiarraymodule.c | 42 ++++++-------------- numpy/core/tests/test_multiarray.py | 37 ----------------- numpy/core/tests/test_umath.py | 13 +++--- 5 files changed, 42 insertions(+), 94 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 724d9ac115b8..bbf529c796c6 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -548,16 +548,16 @@ classes, we will provide a mixin class that provides overrides for all binary operators. -Extension to other numpy functions ----------------------------------- +Future extensions to other functions +------------------------------------ -The ``__array_ufunc__`` method is also used to override -:func:`~numpy.matmul`, since while this function is not a Ufunc, it is -very similar. Indeed, :func:`~numpy.matmul` may well be implemented as -a (generalized) Ufunc in the future, as may happen with some other -functions, such as :func:`~numpy.median`, :func:`~numpy.min`, -:func:`~numpy.argsort`, etc. (in which case it will thus become possible -to override these as well). +Some numpy functions could be implemented as (generalized) Ufunc, in +which case it would be possible for them to be overridden by the +``__array_ufunc__`` method. A prime candidate is :func:`~numpy.matmul`, +which currently is not a Ufunc, but could be relatively easily be +rewritten as a (set of) generalized Ufuncs. The same may happen with +functions such as :func:`~numpy.median`, :func:`~numpy.min`, and +:func:`~numpy.argsort`. Demo ==== @@ -576,16 +576,17 @@ proposed in this NEP. Here is a demo highlighting the functionality.:: In [4]: b = B() - In [5]: np.matmul(a, b) + In [5]: np.negative(b) Out[5]: 'B' In [6]: np.multiply(a, b) Out[6]: 'B' -As a simple example, one could add the following ``__array_ufunc__`` to -SciPy's sparse matrices (just for ``np.matmul`` and ``np.multiply`` as -these are the two most common cases where users would attempt to use -sparse matrices with ufuncs):: +As a simple example, once ``np.matmul`` is covered as well (see above), +one could add the following ``__array_ufunc__`` to SciPy's sparse +matrices (just for ``np.matmul`` and ``np.multiply`` as these are the +two most common cases where users would attempt to use sparse matrices +with ufuncs):: def __array_ufunc__(self, func, method, pos, inputs, **kwargs): """Method for compatibility with NumPy's ufuncs and matmul diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 10a776e8dfdc..219f9ca6490f 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -73,13 +73,14 @@ NumPy provides several hooks that classes can customize: :func:`__array_ufunc__` operations return :obj:`NotImplemented`, a :exc:`TypeError` is raised. - .. note:: In addition to ufuncs, :func:`__array_ufunc__` also - overrides the behavior of :func:`numpy.matmul`, even though it - is not a ufunc. But it can be thought of as :ref:`generalized - universal functions` (which are - overridden). Indeed, we intend to rewrite :func:`numpy.matmul` - as such, and possibly do the same to other relevant functions, - such as :func:`numpy.median`, :func:`numpy.argsort`, etc. + .. note:: We intend to re-implement numpy functions as (generalized) + Ufunc, in which case it will become possible for them to be + overridden by the ``__array_ufunc__`` method. A prime candidate is + :func:`~numpy.matmul`, which currently is not a Ufunc, but could be + relatively easily be rewritten as a (set of) generalized Ufuncs. The + same may happen with functions such as :func:`~numpy.median`, + :func:`~numpy.min`, and :func:`~numpy.argsort`. + Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 4cc2b481c9ce..73ba8c5c4ee9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2187,7 +2187,7 @@ array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) kwlist, &a, &v, &o)) { return NULL; } - if (o) { + if (o != NULL) { if (o == Py_None) { o = NULL; } @@ -2327,14 +2327,10 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *args) * out: Either NULL, or an array into which the output should be placed. * * Returns NULL on error. - * Returns NotImplemented on priority override. */ static PyObject * array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) { - static PyObject *matmul = NULL; - int errval; - PyObject *override = NULL; PyObject *in1, *in2, *out = NULL; char* kwlist[] = {"a", "b", "out", NULL }; PyArrayObject *ap1, *ap2, *ret = NULL; @@ -2345,39 +2341,25 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) char *subscripts; PyArrayObject *ops[2]; - npy_cache_import("numpy.core.multiarray", "matmul", &matmul); - if (matmul == NULL) { - return NULL; - } - - errval = PyUFunc_CheckOverride((PyUFuncObject*)matmul, "__call__", - args, kwds, &override, 2, 1); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matmul", kwlist, &in1, &in2, &out)) { return NULL; } - if (out == Py_None) { - out = NULL; - } - if (out != NULL && !PyArray_Check(out)) { - PyErr_SetString(PyExc_TypeError, - "'out' must be an array"); - return NULL; + if (out != NULL) { + if (out == Py_None) { + out = NULL; + } + else if (!PyArray_Check(out)) { + PyErr_SetString(PyExc_TypeError, "'out' must be an array"); + return NULL; + } } dtype = PyArray_DescrFromObject(in1, NULL); dtype = PyArray_DescrFromObject(in2, dtype); if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, - "Cannot find a common data type."); + PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); return NULL; } typenum = dtype->type_num; @@ -2385,7 +2367,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (typenum == NPY_OBJECT) { /* matmul is not currently implemented for object arrays */ PyErr_SetString(PyExc_TypeError, - "Object arrays are not currently supported"); + "Object arrays are not currently supported"); Py_DECREF(dtype); return NULL; } @@ -2407,7 +2389,7 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) if (PyArray_NDIM(ap1) == 0 || PyArray_NDIM(ap2) == 0) { /* Scalars are rejected */ PyErr_SetString(PyExc_ValueError, - "Scalar operands are not allowed, use '*' instead"); + "Scalar operands are not allowed, use '*' instead"); return NULL; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index bed437f40375..6d9a8fdc3c47 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -5254,43 +5254,6 @@ def test_matrix_matrix_values(self): res = self.matmul(m12, m21) assert_equal(res, tgt12_21) - def test_array_ufunc_override(self): - - class B(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - return "B" - - class C(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - return NotImplemented - - class D(np.ndarray): - def __new__(cls, *args, **kwargs): - return np.array(*args, **kwargs).view(cls) - - __array_ufunc__ = None - - a = np.ones(2) - b = B([1, 2]) - c = C([1, 2]) - d = D([1, 2]) - assert_equal(self.matmul(b, a), "B") - assert_equal(self.matmul(a, b), "B") - assert_equal(self.matmul(b, c), "B") - assert_equal(self.matmul(c, b), "B") - assert_equal(self.matmul(b, d), "B") - assert_equal(self.matmul(d, b), "B") - assert_raises(TypeError, self.matmul, a, c) - assert_raises(TypeError, self.matmul, c, a) - assert_raises(TypeError, self.matmul, d, a) - assert_raises(TypeError, self.matmul, a, d) - class TestMatmul(MatmulCommon, TestCase): matmul = np.matmul diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 1fef5f50f604..5ae4739bb4fc 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1577,19 +1577,19 @@ def __array_ufunc__(self, func, method, *inputs, **kwargs): a = A() b = np.matrix([1]) res0 = np.multiply(a, b) - res1 = np.matmul(a, b) + res1 = np.multiply(b, b, out=a) # self assert_equal(res0[0], a) assert_equal(res1[0], a) assert_equal(res0[1], np.multiply) - assert_equal(res1[1], np.matmul) + assert_equal(res1[1], np.multiply) assert_equal(res0[2], '__call__') assert_equal(res1[2], '__call__') assert_equal(res0[3], (a, b)) - assert_equal(res1[3], (a, b)) + assert_equal(res1[3], (b, b)) assert_equal(res0[4], {}) - assert_equal(res1[4], {}) + assert_equal(res1[4], {'out': (a,)}) def test_ufunc_override_mro(self): @@ -1878,8 +1878,9 @@ def __array_ufunc__(self, *a, **kwargs): raise ValueError("oops") a = A() - for func in [np.divide, np.matmul]: - assert_raises(ValueError, func, a, a) + assert_raises(ValueError, np.negative, 1, out=a) + assert_raises(ValueError, np.negative, a) + assert_raises(ValueError, np.divide, 1., a) def test_gufunc_override(self): # gufunc are just ufunc instances, but follow a different path, From 55500b90c0d868621feb04920782109a57d40c12 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Thu, 6 Apr 2017 12:27:35 -0400 Subject: [PATCH 28/43] MAINT: simplify now that __array_ufunc__ overrides ufuncs only. In particular, use fact that we're guaranteed to have a PyUFuncObject in PyUFunc_CheckOverride. --- numpy/core/src/private/ufunc_override.c | 17 +++++++---------- numpy/core/src/private/ufunc_override.h | 3 +-- numpy/core/src/umath/ufunc_object.c | 18 ++++++------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index 14cf9b39b057..d99a22a5d455 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -11,12 +11,13 @@ static int normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds, - int nin, int nout) + PyObject **normal_args, PyObject **normal_kwds) { /* ufunc.__call__(*args, **kwds) */ int i; int not_all_none; + int nin = ufunc->nin; + int nout = ufunc->nout; int nargs = PyTuple_GET_SIZE(args); PyObject *obj; @@ -325,17 +326,11 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, * * Returns 0 on success and 1 on exception. On success, *result contains the * result of the operation, if any. If *result is NULL, there is no override. - * - * TODO: the ufunc really should always be a ufunc, so that we can rely on - * using, e.g., ufunc->nin, ufunc->nout, etc. Right now, we cannot, since we - * also use this function to override np.dot and np.matmul. This should be - * fixed. */ NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, - PyObject **result, - int nin, int nout) + PyObject **result) { int i; int j; @@ -376,6 +371,8 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, normal_kwds = PyDict_Copy(kwds); out = PyDict_GetItemString(normal_kwds, "out"); if (out != NULL) { + int nout = ufunc->nout; + if (PyTuple_Check(out)) { int all_none = 1; @@ -449,7 +446,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* ufunc.__call__ */ if (strcmp(method, "__call__") == 0) { status = normalize___call___args(ufunc, args, &normal_args, - &normal_kwds, nin, nout); + &normal_kwds); } /* ufunc.reduce and ufunc.accumulate */ else if ((strcmp(method, "reduce") == 0) || diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 68dd0221d0b0..92618453b492 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -11,6 +11,5 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyObject *args, PyObject *kwds, - PyObject **result, - int nin, int nout); + PyObject **result); #endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 605d59e617a5..04aee4aef4d4 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -4370,8 +4370,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) mps[i] = NULL; } - errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override, - ufunc->nin, ufunc->nout); + errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override); if (errval) { return NULL; } @@ -5088,8 +5087,7 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) } /* Note: `nin` and `nout` are not used in the normalization */ - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override, - ufunc->nin, ufunc->nout); + errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); if (errval) { return NULL; } @@ -5167,8 +5165,7 @@ ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *override = NULL; /* `nin` and `nout`, the last two arguments, are not actually used */ - errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, - 1, ufunc->nout); + errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override); if (errval) { return NULL; } @@ -5185,8 +5182,7 @@ ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *override = NULL; /* `nin` and `nout`, the last two arguments, are not actually used */ - errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, - 1, ufunc->nout); + errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override); if (errval) { return NULL; } @@ -5203,8 +5199,7 @@ ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *override = NULL; /* `nin` and `nout`, the last two arguments, are not actually used */ - errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, - ufunc->nin, ufunc->nout); + errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override); if (errval) { return NULL; } @@ -5269,8 +5264,7 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) NPY_BEGIN_THREADS_DEF; /* `nin` and `nout`, the last two arguments, are not actually used */ - errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override, - ufunc->nin + 1, 0); + errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override); if (errval) { return NULL; } From 25e973d61150f515448566d35a86ea878aa4c98f Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Thu, 6 Apr 2017 13:15:04 -0400 Subject: [PATCH 29/43] MAINT: split out umath-specific part of ufunc_override. And put it in src/umath instead of src/private. This leaves only the PyUFunc_WithOverride function in ufunc_override.c, which is all that is needed outside of umath (it is called by array_ufunc in multiarray/methods.c). --- numpy/core/setup.py | 2 + numpy/core/src/multiarray/methods.c | 2 +- numpy/core/src/private/ufunc_override.c | 478 +----------------------- numpy/core/src/private/ufunc_override.h | 16 +- numpy/core/src/umath/override.c | 478 ++++++++++++++++++++++++ numpy/core/src/umath/override.h | 11 + numpy/core/src/umath/ufunc_object.c | 2 +- 7 files changed, 509 insertions(+), 480 deletions(-) create mode 100644 numpy/core/src/umath/override.c create mode 100644 numpy/core/src/umath/override.h diff --git a/numpy/core/setup.py b/numpy/core/setup.py index b29a8fee2c27..e057c56141d3 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -874,6 +874,7 @@ def generate_umath_c(ext, build_dir): join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'scalarmath.c.src'), join('src', 'umath', 'ufunc_type_resolution.c'), + join('src', 'umath', 'override.c'), join('src', 'private', 'mem_overlap.c'), join('src', 'private', 'ufunc_override.c')] @@ -884,6 +885,7 @@ def generate_umath_c(ext, build_dir): join('src', 'multiarray', 'common.h'), join('src', 'private', 'templ_common.h.src'), join('src', 'umath', 'simd.inc.src'), + join('src', 'umath', 'override.h'), join(codegen_dir, 'generate_ufunc_api.py'), join('src', 'private', 'lowlevel_strided_loops.h'), join('src', 'private', 'mem_overlap.h'), diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 679e723a6e8d..26f98e006a35 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1023,7 +1023,7 @@ array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } /* ndarray cannot handle overrides itself */ - if (PyUFunc_HasOverride(normal_args, kwds, NULL)) { + if (PyUFunc_WithOverride(normal_args, kwds, NULL)) { result = Py_NotImplemented; Py_INCREF(Py_NotImplemented); goto cleanup; diff --git a/numpy/core/src/private/ufunc_override.c b/numpy/core/src/private/ufunc_override.c index d99a22a5d455..b5cd46b898f5 100644 --- a/numpy/core/src/private/ufunc_override.c +++ b/numpy/core/src/private/ufunc_override.c @@ -2,201 +2,11 @@ #define NO_IMPORT_ARRAY #include "npy_pycompat.h" -#include "numpy/ufuncobject.h" #include "get_attr_string.h" #include "npy_import.h" #include "ufunc_override.h" - -static int -normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.__call__(*args, **kwds) */ - int i; - int not_all_none; - int nin = ufunc->nin; - int nout = ufunc->nout; - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - - if (nargs < nin) { - PyErr_Format(PyExc_TypeError, - "required input argument (pos %d) not found", nin); - return -1; - } - if (nargs > nin+nout) { - PyErr_Format(PyExc_TypeError, - "ufunc takes at most %d arguments (%d given)", - nin+nout, nargs); - return -1; - } - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - obj = PyDict_GetItemString(*normal_kwds, "sig"); - if (obj != NULL) { - if (PyDict_GetItemString(*normal_kwds, "signature")) { - PyErr_SetString(PyExc_TypeError, - "cannot specify both 'sig' and 'signature'"); - return -1; - } - Py_INCREF(obj); - PyDict_SetItemString(*normal_kwds, "signature", obj); - PyDict_DelItemString(*normal_kwds, "sig"); - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - if (*normal_args == NULL) { - return -1; - } - - /* If we have more args than nin, they must be the output variables.*/ - if (nargs > nin) { - if(PyDict_GetItemString(*normal_kwds, "out")) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('out') and position (%d)", - nin); - return -1; - } - for (i=nin; i < nargs; i++) { - not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); - if (not_all_none) { - break; - } - } - if (not_all_none) { - if (nargs - nin == nout) - { - obj = PyTuple_GetSlice(args, nin, nargs); - } - else { - PyObject *item; - - obj = PyTuple_New(nout); - if (obj == NULL) { - return -1; - } - for (i = 0; i < nout; i++) { - if (i + nin < nargs) { - item = PyTuple_GET_ITEM(args, nin+i); - } - else { - item = Py_None; - } - Py_INCREF(item); - PyTuple_SET_ITEM(obj, i, item); - } - } - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } - } - return 0; -} - -static int -normalize_reduce_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* - * ufunc.reduce(a[, axis, dtype, out, keepdims]) - * ufunc.accumulate(a[, axis, dtype, out]) - * the number of arguments has been checked in PyUFunc_GenericReduction. - */ - int nargs = PyTuple_GET_SIZE(args); - int i; - PyObject *obj; - static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; - - *normal_args = PyTuple_GetSlice(args, 0, 1); - if (*normal_args == NULL) { - return -1; - } - - for (i = 1; i < nargs; i++) { - if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('%s') and position (%d)", - kwlist[i], i); - return -1; - } - obj = PyTuple_GET_ITEM(args, i); - if (obj != Py_None) { - if (i == 3) { - obj = PyTuple_GetSlice(args, 3, 4); - } - PyDict_SetItemString(*normal_kwds, kwlist[i], obj); - if (i == 3) { - Py_DECREF(obj); - } - } - } - return 0; -} - -static int -normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* - * ufunc.reduceat(a, indicies[, axis, dtype, out]) - * the number of arguments has been checked in PyUFunc_GenericReduction. - */ - int i; - int nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; - - /* a and indicies */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - if (*normal_args == NULL) { - return -1; - } - - for (i = 2; i < nargs; i++) { - if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('%s') and position (%d)", - kwlist[i], i); - return -1; - } - obj = PyTuple_GET_ITEM(args, i); - if (obj != Py_None) { - if (i == 4) { - obj = PyTuple_GetSlice(args, 4, 5); - } - PyDict_SetItemString(*normal_kwds, kwlist[i], obj); - if (i == 4) { - Py_DECREF(obj); - } - } - } - return 0; -} - -static int -normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* - * ufunc.outer(A, B) - * This has no kwds so we don't need to do any kwd stuff. - */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - return (*normal_args == NULL); -} - -static int -normalize_at_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.at(a, indices[, b]) */ - int nargs = PyTuple_GET_SIZE(args); - - *normal_args = PyTuple_GetSlice(args, 0, nargs); - return (*normal_args == NULL); -} - /* * Check whether an object has __array_ufunc__ defined on its class and it * is not the default, i.e., the object is not an ndarray, and its @@ -240,13 +50,15 @@ has_non_default_array_ufunc(PyObject *obj) /* * Check whether a set of input and output args have a non-default - * `__array_ufunc__` method. Returns the number of overrides, setting - * corresponding objects in PyObject array with_override (if not NULL). + * `__array_ufunc__` method. Return the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL) + * using borrowed references. + * * returns -1 on failure. */ NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds, - PyObject **with_override) +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override) { int i; @@ -262,7 +74,7 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, */ if (!PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, - "Internal Numpy error: call to PyUFunc_CheckOverride " + "Internal Numpy error: call to PyUFunc_HasOverride " "with non-tuple"); goto fail; } @@ -270,7 +82,7 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, if (nargs > NPY_MAXARGS) { PyErr_SetString(PyExc_TypeError, "Internal Numpy error: too many arguments in call " - "to PyUFunc_CheckOverride"); + "to PyUFunc_HasOverride"); goto fail; } /* be sure to include possible 'out' keyword argument. */ @@ -316,277 +128,3 @@ PyUFunc_HasOverride(PyObject *args, PyObject *kwds, fail: return -1; } -/* - * Check a set of args for the `__array_ufunc__` method. If more than one of - * the input arguments implements `__array_ufunc__`, they are tried in the - * order: subclasses before superclasses, otherwise left to right. The first - * (non-None) routine returning something other than `NotImplemented` - * determines the result. If all of the `__array_ufunc__` operations return - * `NotImplemented` (or are None), a `TypeError` is raised. - * - * Returns 0 on success and 1 on exception. On success, *result contains the - * result of the operation, if any. If *result is NULL, there is no override. - */ -NPY_NO_EXPORT int -PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result) -{ - int i; - int j; - int status; - - int noa; - PyObject *with_override[NPY_MAXARGS]; - - PyObject *obj; - PyObject *other_obj; - PyObject *out; - - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ - PyObject *normal_kwds = NULL; - - PyObject *override_args = NULL; - Py_ssize_t len; - - /* - * Check inputs for overrides - */ - noa = PyUFunc_HasOverride(args, kwds, with_override); - /* No overrides, bail out.*/ - if (noa == 0) { - *result = NULL; - return 0; - } - - /* - * Normalize ufunc arguments. - */ - - /* Build new kwds */ - if (kwds && PyDict_CheckExact(kwds)) { - - /* ensure out is always a tuple */ - normal_kwds = PyDict_Copy(kwds); - out = PyDict_GetItemString(normal_kwds, "out"); - if (out != NULL) { - int nout = ufunc->nout; - - if (PyTuple_Check(out)) { - int all_none = 1; - - if (PyTuple_GET_SIZE(out) != nout) { - PyErr_Format(PyExc_TypeError, - "The 'out' tuple must have exactly " - "%d entries: one per ufunc output", nout); - goto fail; - } - for (i = 0; i < PyTuple_GET_SIZE(out); i++) { - all_none = (PyTuple_GET_ITEM(out, i) == Py_None); - if (!all_none) { - break; - } - } - if (all_none) { - PyDict_DelItemString(normal_kwds, "out"); - } - } - else { - /* not a tuple */ - if (nout > 1 && DEPRECATE("passing a single argument to the " - "'out' keyword argument of a " - "ufunc with\n" - "more than one output will " - "result in an error in the " - "future") < 0) { - /* - * If the deprecation is removed, also remove the loop - * below setting tuple items to None (but keep this future - * error message.) - */ - PyErr_SetString(PyExc_TypeError, - "'out' must be a tuple of arguments"); - goto fail; - } - if (out != Py_None) { - /* not already a tuple and not None */ - PyObject *out_tuple = PyTuple_New(nout); - - if (out_tuple == NULL) { - goto fail; - } - for (i = 1; i < nout; i++) { - Py_INCREF(Py_None); - PyTuple_SET_ITEM(out_tuple, i, Py_None); - } - /* out was borrowed ref; make it permanent */ - Py_INCREF(out); - /* steals reference */ - PyTuple_SET_ITEM(out_tuple, 0, out); - PyDict_SetItemString(normal_kwds, "out", out_tuple); - Py_DECREF(out_tuple); - } - else { - /* out=None; remove it */ - PyDict_DelItemString(normal_kwds, "out"); - } - } - } - } - else { - normal_kwds = PyDict_New(); - } - if (normal_kwds == NULL) { - goto fail; - } - - /* decide what to do based on the method. */ - - /* ufunc.__call__ */ - if (strcmp(method, "__call__") == 0) { - status = normalize___call___args(ufunc, args, &normal_args, - &normal_kwds); - } - /* ufunc.reduce and ufunc.accumulate */ - else if ((strcmp(method, "reduce") == 0) || - (strcmp(method, "accumulate") == 0)) { - status = normalize_reduce_accumulate_args(ufunc, args, &normal_args, - &normal_kwds); - } - /* ufunc.reduceat */ - else if (strcmp(method, "reduceat") == 0) { - status = normalize_reduceat_args(ufunc, args, &normal_args, - &normal_kwds); - } - /* ufunc.outer */ - else if (strcmp(method, "outer") == 0) { - status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); - } - /* ufunc.at */ - else if (strcmp(method, "at") == 0) { - status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); - } - /* unknown method */ - else { - PyErr_Format(PyExc_TypeError, - "Internal Numpy error: unknown ufunc method '%s' in call " - "to PyUFunc_CheckOverride", method); - status = -1; - } - if (status != 0) { - Py_XDECREF(normal_args); - goto fail; - } - - len = PyTuple_GET_SIZE(normal_args); - override_args = PyTuple_New(len + 2); - if (override_args == NULL) { - goto fail; - } - - Py_INCREF(ufunc); - /* PyTuple_SET_ITEM steals reference */ - PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); - method_name = PyUString_FromString(method); - if (method_name == NULL) { - goto fail; - } - Py_INCREF(method_name); - PyTuple_SET_ITEM(override_args, 1, method_name); - for (i = 0; i < len; i++) { - PyObject *item = PyTuple_GET_ITEM(normal_args, i); - - Py_INCREF(item); - PyTuple_SET_ITEM(override_args, i + 2, item); - } - Py_DECREF(normal_args); - - /* Call __array_ufunc__ functions in correct order */ - while (1) { - PyObject *array_ufunc; - PyObject *override_obj; - - override_obj = NULL; - *result = NULL; - - /* Choose an overriding argument */ - for (i = 0; i < noa; i++) { - obj = with_override[i]; - if (obj == NULL) { - continue; - } - - /* Get the first instance of an overriding arg.*/ - override_obj = obj; - - /* Check for sub-types to the right of obj. */ - for (j = i + 1; j < noa; j++) { - other_obj = with_override[j]; - if (PyObject_Type(other_obj) != PyObject_Type(obj) && - PyObject_IsInstance(other_obj, - PyObject_Type(override_obj))) { - override_obj = NULL; - break; - } - } - - /* override_obj had no subtypes to the right. */ - if (override_obj) { - /* We won't call this one again */ - with_override[i] = NULL; - break; - } - } - - /* Check if there is a method left to call */ - if (!override_obj) { - /* No acceptable override found. */ - PyErr_SetString(PyExc_TypeError, - "__array_ufunc__ not implemented for this type."); - goto fail; - } - - /* Access the override */ - array_ufunc = PyObject_GetAttrString(override_obj, - "__array_ufunc__"); - if (array_ufunc == NULL) { - goto fail; - } - - /* If None, try next one (i.e., as if it returned NotImplemented) */ - if (array_ufunc == Py_None) { - Py_DECREF(array_ufunc); - continue; - } - - *result = PyObject_Call(array_ufunc, override_args, normal_kwds); - Py_DECREF(array_ufunc); - - if (*result == NULL) { - /* Exception occurred */ - goto fail; - } - else if (*result == Py_NotImplemented) { - /* Try the next one */ - Py_DECREF(*result); - continue; - } - else { - /* Good result. */ - break; - } - } - - /* Override found, return it. */ - Py_XDECREF(method_name); - Py_XDECREF(normal_kwds); - Py_DECREF(override_args); - return 0; - -fail: - Py_XDECREF(method_name); - Py_XDECREF(normal_kwds); - Py_XDECREF(override_args); - return 1; -} diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h index 92618453b492..15a932174cfc 100644 --- a/numpy/core/src/private/ufunc_override.h +++ b/numpy/core/src/private/ufunc_override.h @@ -2,14 +2,14 @@ #define __UFUNC_OVERRIDE_H #include "npy_config.h" -#include "numpy/ufuncobject.h" +/* + * Check whether a set of input and output args have a non-default + * `__array_ufunc__` method. Returns the number of overrides, setting + * corresponding objects in PyObject array with_override (if not NULL). + * returns -1 on failure. + */ NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds, - PyObject **with_override); - -NPY_NO_EXPORT int -PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result); +PyUFunc_WithOverride(PyObject *args, PyObject *kwds, + PyObject **with_override); #endif diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c new file mode 100644 index 000000000000..61a6bb720c62 --- /dev/null +++ b/numpy/core/src/umath/override.c @@ -0,0 +1,478 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NO_IMPORT_ARRAY + +#include "npy_pycompat.h" +#include "numpy/ufuncobject.h" +#include "npy_import.h" + +#include "ufunc_override.h" +#include "override.h" + +/* + * The following functions normalize ufunc arguments. The work done is similar + * to what is done inside ufunc_object by get_ufunc_arguments for __call__ and + * generalized ufuncs, and by PyUFunc_GenericReduction for the other methods. + * It would be good to unify (see gh-8892). + */ +static int +normalize___call___args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.__call__(*args, **kwds) */ + int i; + int not_all_none; + int nin = ufunc->nin; + int nout = ufunc->nout; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "required input argument (pos %d) not found", nin); + return -1; + } + if (nargs > nin+nout) { + PyErr_Format(PyExc_TypeError, + "ufunc takes at most %d arguments (%d given)", + nin+nout, nargs); + return -1; + } + /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + obj = PyDict_GetItemString(*normal_kwds, "sig"); + if (obj != NULL) { + if (PyDict_GetItemString(*normal_kwds, "signature")) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + return -1; + } + Py_INCREF(obj); + PyDict_SetItemString(*normal_kwds, "signature", obj); + PyDict_DelItemString(*normal_kwds, "sig"); + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* If we have more args than nin, they must be the output variables.*/ + if (nargs > nin) { + if(PyDict_GetItemString(*normal_kwds, "out")) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('out') and position (%d)", + nin); + return -1; + } + for (i=nin; i < nargs; i++) { + not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); + if (not_all_none) { + break; + } + } + if (not_all_none) { + if (nargs - nin == nout) + { + obj = PyTuple_GetSlice(args, nin, nargs); + } + else { + PyObject *item; + + obj = PyTuple_New(nout); + if (obj == NULL) { + return -1; + } + for (i = 0; i < nout; i++) { + if (i + nin < nargs) { + item = PyTuple_GET_ITEM(args, nin+i); + } + else { + item = Py_None; + } + Py_INCREF(item); + PyTuple_SET_ITEM(obj, i, item); + } + } + PyDict_SetItemString(*normal_kwds, "out", obj); + Py_DECREF(obj); + } + } + return 0; +} + +static int +normalize_reduce_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduce(a[, axis, dtype, out, keepdims]) + * ufunc.accumulate(a[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.reduceat(a, indicies[, axis, dtype, out]) + * the number of arguments has been checked in PyUFunc_GenericReduction. + */ + int i; + int nargs = PyTuple_GET_SIZE(args); + PyObject *obj; + static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; + + /* a and indicies */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + if (*normal_args == NULL) { + return -1; + } + + for (i = 2; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 4) { + obj = PyTuple_GetSlice(args, 4, 5); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 4) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* + * ufunc.outer(A, B) + * This has no kwds so we don't need to do any kwd stuff. + */ + *normal_args = PyTuple_GetSlice(args, 0, 2); + return (*normal_args == NULL); +} + +static int +normalize_at_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* ufunc.at(a, indices[, b]) */ + int nargs = PyTuple_GET_SIZE(args); + + *normal_args = PyTuple_GetSlice(args, 0, nargs); + return (*normal_args == NULL); +} + +/* + * Check a set of args for the `__array_ufunc__` method. If more than one of + * the input arguments implements `__array_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * (non-None) routine returning something other than `NotImplemented` + * determines the result. If all of the `__array_ufunc__` operations return + * `NotImplemented` (or are None), a `TypeError` is raised. + * + * Returns 0 on success and 1 on exception. On success, *result contains the + * result of the operation, if any. If *result is NULL, there is no override. + */ +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result) +{ + int i; + int j; + int status; + + int noa; + PyObject *with_override[NPY_MAXARGS]; + + PyObject *obj; + PyObject *other_obj; + PyObject *out; + + PyObject *method_name = NULL; + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + + PyObject *override_args = NULL; + Py_ssize_t len; + + /* + * Check inputs for overrides + */ + noa = PyUFunc_WithOverride(args, kwds, with_override); + /* No overrides, bail out.*/ + if (noa == 0) { + *result = NULL; + return 0; + } + + /* + * Normalize ufunc arguments. + */ + + /* Build new kwds */ + if (kwds && PyDict_CheckExact(kwds)) { + + /* ensure out is always a tuple */ + normal_kwds = PyDict_Copy(kwds); + out = PyDict_GetItemString(normal_kwds, "out"); + if (out != NULL) { + int nout = ufunc->nout; + + if (PyTuple_Check(out)) { + int all_none = 1; + + if (PyTuple_GET_SIZE(out) != nout) { + PyErr_Format(PyExc_TypeError, + "The 'out' tuple must have exactly " + "%d entries: one per ufunc output", nout); + goto fail; + } + for (i = 0; i < PyTuple_GET_SIZE(out); i++) { + all_none = (PyTuple_GET_ITEM(out, i) == Py_None); + if (!all_none) { + break; + } + } + if (all_none) { + PyDict_DelItemString(normal_kwds, "out"); + } + } + else { + /* not a tuple */ + if (nout > 1 && DEPRECATE("passing a single argument to the " + "'out' keyword argument of a " + "ufunc with\n" + "more than one output will " + "result in an error in the " + "future") < 0) { + /* + * If the deprecation is removed, also remove the loop + * below setting tuple items to None (but keep this future + * error message.) + */ + PyErr_SetString(PyExc_TypeError, + "'out' must be a tuple of arguments"); + goto fail; + } + if (out != Py_None) { + /* not already a tuple and not None */ + PyObject *out_tuple = PyTuple_New(nout); + + if (out_tuple == NULL) { + goto fail; + } + for (i = 1; i < nout; i++) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(out_tuple, i, Py_None); + } + /* out was borrowed ref; make it permanent */ + Py_INCREF(out); + /* steals reference */ + PyTuple_SET_ITEM(out_tuple, 0, out); + PyDict_SetItemString(normal_kwds, "out", out_tuple); + Py_DECREF(out_tuple); + } + else { + /* out=None; remove it */ + PyDict_DelItemString(normal_kwds, "out"); + } + } + } + } + else { + normal_kwds = PyDict_New(); + } + if (normal_kwds == NULL) { + goto fail; + } + + /* decide what to do based on the method. */ + + /* ufunc.__call__ */ + if (strcmp(method, "__call__") == 0) { + status = normalize___call___args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduce and ufunc.accumulate */ + else if ((strcmp(method, "reduce") == 0) || + (strcmp(method, "accumulate") == 0)) { + status = normalize_reduce_accumulate_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.reduceat */ + else if (strcmp(method, "reduceat") == 0) { + status = normalize_reduceat_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.outer */ + else if (strcmp(method, "outer") == 0) { + status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + } + /* ufunc.at */ + else if (strcmp(method, "at") == 0) { + status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + } + /* unknown method */ + else { + PyErr_Format(PyExc_TypeError, + "Internal Numpy error: unknown ufunc method '%s' in call " + "to PyUFunc_CheckOverride", method); + status = -1; + } + if (status != 0) { + Py_XDECREF(normal_args); + goto fail; + } + + len = PyTuple_GET_SIZE(normal_args); + override_args = PyTuple_New(len + 2); + if (override_args == NULL) { + goto fail; + } + + Py_INCREF(ufunc); + /* PyTuple_SET_ITEM steals reference */ + PyTuple_SET_ITEM(override_args, 0, (PyObject *)ufunc); + method_name = PyUString_FromString(method); + if (method_name == NULL) { + goto fail; + } + Py_INCREF(method_name); + PyTuple_SET_ITEM(override_args, 1, method_name); + for (i = 0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(normal_args, i); + + Py_INCREF(item); + PyTuple_SET_ITEM(override_args, i + 2, item); + } + Py_DECREF(normal_args); + + /* Call __array_ufunc__ functions in correct order */ + while (1) { + PyObject *array_ufunc; + PyObject *override_obj; + + override_obj = NULL; + *result = NULL; + + /* Choose an overriding argument */ + for (i = 0; i < noa; i++) { + obj = with_override[i]; + if (obj == NULL) { + continue; + } + + /* Get the first instance of an overriding arg.*/ + override_obj = obj; + + /* Check for sub-types to the right of obj. */ + for (j = i + 1; j < noa; j++) { + other_obj = with_override[j]; + if (PyObject_Type(other_obj) != PyObject_Type(obj) && + PyObject_IsInstance(other_obj, + PyObject_Type(override_obj))) { + override_obj = NULL; + break; + } + } + + /* override_obj had no subtypes to the right. */ + if (override_obj) { + /* We won't call this one again */ + with_override[i] = NULL; + break; + } + } + + /* Check if there is a method left to call */ + if (!override_obj) { + /* No acceptable override found. */ + PyErr_SetString(PyExc_TypeError, + "__array_ufunc__ not implemented for this type."); + goto fail; + } + + /* Access the override */ + array_ufunc = PyObject_GetAttrString(override_obj, + "__array_ufunc__"); + if (array_ufunc == NULL) { + goto fail; + } + + /* If None, try next one (i.e., as if it returned NotImplemented) */ + if (array_ufunc == Py_None) { + Py_DECREF(array_ufunc); + continue; + } + + *result = PyObject_Call(array_ufunc, override_args, normal_kwds); + Py_DECREF(array_ufunc); + + if (*result == NULL) { + /* Exception occurred */ + goto fail; + } + else if (*result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(*result); + continue; + } + else { + /* Good result. */ + break; + } + } + + /* Override found, return it. */ + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_DECREF(override_args); + return 0; + +fail: + Py_XDECREF(method_name); + Py_XDECREF(normal_kwds); + Py_XDECREF(override_args); + return 1; +} diff --git a/numpy/core/src/umath/override.h b/numpy/core/src/umath/override.h new file mode 100644 index 000000000000..68f3c6ef0814 --- /dev/null +++ b/numpy/core/src/umath/override.h @@ -0,0 +1,11 @@ +#ifndef _NPY_UMATH_OVERRIDE_H +#define _NPY_UMATH_OVERRIDE_H + +#include "npy_config.h" +#include "numpy/ufuncobject.h" + +NPY_NO_EXPORT int +PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result); +#endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 04aee4aef4d4..1bc7e1109c8f 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -44,7 +44,7 @@ #include "mem_overlap.h" #include "ufunc_object.h" -#include "ufunc_override.h" +#include "override.h" /********** PRINTF DEBUG TRACING **************/ #define NPY_UF_DBG_TRACING 0 From b1fa10aaf1dc70343e7b267a6e2858ad30b0d97e Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Sat, 8 Apr 2017 11:50:54 -0400 Subject: [PATCH 30/43] BUG: ensure subclass of override class doesn't segfault. --- numpy/core/src/umath/override.c | 3 ++- numpy/core/tests/test_umath.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index 61a6bb720c62..6cd4bee14b7e 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -409,7 +409,8 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Check for sub-types to the right of obj. */ for (j = i + 1; j < noa; j++) { other_obj = with_override[j]; - if (PyObject_Type(other_obj) != PyObject_Type(obj) && + if (other_obj != NULL && + PyObject_Type(other_obj) != PyObject_Type(obj) && PyObject_IsInstance(other_obj, PyObject_Type(override_obj))) { override_obj = NULL; diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 5ae4739bb4fc..454f2702035f 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -7,6 +7,7 @@ from numpy.testing.utils import _gen_alignment_data import numpy.core.umath as ncu +from numpy.core import umath_tests as ncu_tests import numpy as np from numpy.testing import ( TestCase, run_module_suite, assert_, assert_equal, assert_raises, @@ -1620,7 +1621,7 @@ class C(object): def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented - class CSub(object): + class CSub(C): def __array_ufunc__(self, func, method, *inputs, **kwargs): return NotImplemented @@ -1889,8 +1890,8 @@ class A(object): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return self, ufunc, method, inputs, kwargs + inner1d = ncu_tests.inner1d a = A() - inner1d = np.core.umath_tests.inner1d res = inner1d(a, a) assert_equal(res[0], a) assert_equal(res[1], inner1d) From 1de8f5a0e8a60295b8581455feef40a0e614912d Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sat, 8 Apr 2017 13:35:17 -0600 Subject: [PATCH 31/43] DOC: Mention `__array_ufunc__` in the 1.13.0 release notes. [ci skip] --- doc/release/1.13.0-notes.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/release/1.13.0-notes.rst b/doc/release/1.13.0-notes.rst index 2ee0b80ae4db..dde2f1fd86c2 100644 --- a/doc/release/1.13.0-notes.rst +++ b/doc/release/1.13.0-notes.rst @@ -7,10 +7,12 @@ This release supports Python 2.7 and 3.4 - 3.6. Highlights ========== - * Reuse of temporaries, operations like ``a + b + c`` will create fewer - temporaries on some platforms. + * Operations like ``a + b + c`` will reuse temporaries on some platforms, + resulting less memory use and faster execution. * Inplace operations check if inputs overlap outputs and create temporaries to avoid problems. + * Improved ability for classes to override default ufunc behavior. See + ``__array_ufunc__`` below. Dropped Support @@ -96,6 +98,16 @@ used instead. New Features ============ +``__array_ufunc__`` added +------------------------- +This is the renamed and redesigned ``__numpy_ufunc__``. Any class, ndarray +subclass or not, can define this method or set it to ``None`` in order to +override the behavior of NumPy's ufuncs. This works quite similarly to Python's +``__mul__`` and other binary operation routines. See the documentation for a +more detailed description of the implementation and behavior of this new +option. The API is provisional, we do not yet guarantee backward compatibility +as modifications may be made pending feedback. + ``PyArray_MapIterArrayCopyIfOverlap`` added to NumPy C-API ---------------------------------------------------------- Similar to ``PyArray_MapIterArray`` but with an additional ``copy_if_overlap`` From a460015bc7d17f0837e112ec10a1d3ca95ac0d18 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 9 Apr 2017 17:19:22 +0200 Subject: [PATCH 32/43] DOC: ufunc-overrides: sync the discussion vs. current implementation --- doc/neps/ufunc-overrides.rst | 132 +++++++++++------------------------ 1 file changed, 41 insertions(+), 91 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index bbf529c796c6..9ef6b220733d 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -27,7 +27,7 @@ objects. e.g. SciPy's sparse matrices [2]_ [3]_. Here we propose adding a mechanism to override ufuncs based on the ufunc checking each of it's arguments for a ``__array_ufunc__`` method. On discovery of ``__array_ufunc__`` the ufunc will hand off the -operation to the method. +operation to the method. This covers some of the same ground as Travis Oliphant's proposal to retro-fit NumPy with multi-methods [4]_, which would solve the same @@ -38,6 +38,14 @@ specifically addresses how binary operators and ufuncs should interact. ``__numpy_ufunc__``. An implementation was made, but had not quite the right behaviour, hence the change in name.) +The ``__array_ufunc__`` as described below requires that any +corresponding Python binary operations (``__mul__`` et al.) should be +implemented in a specific way and be compatible with Numpy's ndarray +semantics. Objects that do not satisfy this cannot override any Numpy +ufuncs. We do not specify a future-compatible path by which this +requirement can be relaxed --- any changes here require corresponding +changes in 3rd party code. + .. [1] http://docs.python.org/doc/numpy/user/basics.subclassing.html .. [2] https://github.com/scipy/scipy/issues/2123 .. [3] https://github.com/scipy/scipy/issues/1569 @@ -117,8 +125,10 @@ scalar, which was then multiplied with all elements of the ``b`` array. However, this behavior is more confusing than useful, and having a :exc:`TypeError` would be preferable. -Adding the ``__array_ufunc__`` functionality fixes this and would -deprecate the other ufunc modifying functions. +This proposal will *not* resolve the issue with scipy.sparse matrices, +which have multiplication semantics incompatible with numpy arrays. +However, the aim is to enable writing other custom array types that have +strictly ndarray compatible semantics. .. [5] http://mail.python.org/pipermail/numpy-discussion/2011-June/056945.html @@ -427,18 +437,32 @@ incompatible relative to :class:`ndarray`. In combination with Python's binary operations ---------------------------------------------- -The ``__array_ufunc__`` mechanism is fully independent of Python's -standard operator override mechanism, and the two do not interact -directly. - -They have indirect interactions, however, because NumPy's -:class:`ndarray` type implements its binary operations via Ufuncs. For -most numerical classes, the easiest way to override binary operations is -thus to define ``__array_ufunc__`` and override the corresponding -Ufunc. The class can then, like :class:`ndarray` itself, define the -binary operators in terms of Ufuncs. Here, one has to take some care to -ensure that one allows for other classes to indicate they are not -compatible, i.e., implementations should be something like:: +The Python operator override mechanism in :class:`ndarray` is coupled to +the ``__array_ufunc__`` mechanism. :class:`ndarray` returns +:obj:`NotImplemented` from ``ndarray.__mul__(self, other)`` and other +binary operation methods if ``other.__array_ufunc__ is None``. If the +``__array_ufunc__`` attribute is absent, :obj:`NotImplemented` is +returned if ``other.__array_priority__ > self.__array_priority__``. In +other cases, :class:`ndarray` calls the corresponding ufunc. The +resulting behavior can modified by overriding the corresponding ufunc +via implementing ``__array_ufunc__``. + +A class wishing to modify the interaction with :class:`ndarray` in +binary operations has two options: + +1. Implement ``__array_ufunc__`` and follow Numpy semantics for Python + binary operations (see below). + +2. Set ``__array_ufunc__ = None``, and implement Python binary + operations freely. In this case, ufuncs will raise :exc:`TypeError` + in combination with ndarray inputs. + +For most numerical classes, the easiest way to override binary +operations is thus to define ``__array_ufunc__`` and override the +corresponding Ufunc. The class can then, like :class:`ndarray` itself, +define the binary operators in terms of Ufuncs. Here, one has to take +some care to ensure that one allows for other classes to indicate they +are not compatible, i.e., implementations should be something like:: class ArrayLike(object): ... @@ -453,6 +477,8 @@ compatible, i.e., implementations should be something like:: return np.multiply(self, other) def __rmul__(self, other): + if getattr(other, '__array_ufunc__', False) is None: + return NotImplemented return np.multiply(other, self) def __imul__(self, other): @@ -559,83 +585,7 @@ rewritten as a (set of) generalized Ufuncs. The same may happen with functions such as :func:`~numpy.median`, :func:`~numpy.min`, and :func:`~numpy.argsort`. -Demo -==== - -A pull request [8]_ has been made including the changes and revisions -proposed in this NEP. Here is a demo highlighting the functionality.:: - - In [1]: import numpy as np; - - In [2]: a = np.array([1]) - - In [3]: class B(): - ...: def __array_ufunc__(self, func, method, pos, inputs, **kwargs): - ...: return "B" - ...: - - In [4]: b = B() - - In [5]: np.negative(b) - Out[5]: 'B' - - In [6]: np.multiply(a, b) - Out[6]: 'B' - -As a simple example, once ``np.matmul`` is covered as well (see above), -one could add the following ``__array_ufunc__`` to SciPy's sparse -matrices (just for ``np.matmul`` and ``np.multiply`` as these are the -two most common cases where users would attempt to use sparse matrices -with ufuncs):: - - def __array_ufunc__(self, func, method, pos, inputs, **kwargs): - """Method for compatibility with NumPy's ufuncs and matmul - functions. - """ - - without_self = list(inputs) - without_self.pop(self) - without_self = tuple(without_self) - - if func is np.multiply: - return self.multiply(*without_self) - - elif func is np.matmul: - if pos == 0: - return self.__mul__(inputs[1]) - if pos == 1: - return self.__rmul__(inputs[0]) - else: - return NotImplemented - -So we now get the expected behavior when using ufuncs with sparse matrices.:: - - In [1]: import numpy as np; import scipy.sparse as sp - - In [2]: a = np.random.randint(3, size=(3,3)) - - In [3]: b = np.random.randint(3, size=(3,3)) - - In [4]: asp = sp.csr_matrix(a); bsp = sp.csr_matrix(b) - - In [5]: np.matmul(a,b) - Out[5]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]]) - - In [6]: np.matmul(asp,b) - Out[6]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]], dtype=int64) - In [7]: np.matmul(asp, bsp).A - Out[7]: - array([[2, 4, 8], - [2, 4, 8], - [2, 2, 3]], dtype=int64) - .. Local Variables: .. mode: rst .. coding: utf-8 From cd2e42c146c5e17a0731a02944ffb65619ae3c7d Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 9 Apr 2017 18:26:56 +0200 Subject: [PATCH 33/43] DOC: ufunc-overrides: revise hierarchy discussion --- doc/neps/ufunc-overrides.rst | 145 ++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 9ef6b220733d..1eaefbf5df5b 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -134,6 +134,7 @@ strictly ndarray compatible semantics. .. [6] https://github.com/numpy/numpy/issues/5844 + Proposed interface ================== @@ -194,65 +195,62 @@ override their inherited ``__array_ufunc__`` implementation. Type casting hierarchy ---------------------- -Similarly to the Python operator dispatch mechanism, writing ufunc -dispatch methods requires some discipline in order to achieve -predictable results. +The Python operator override mechanism gives much freedom in how to +write the override methods, and it requires some discipline in order to +achieve predictable results. Here, we discuss an approach for +understanding some of the implications, which can provide input in the +design. + +It is useful to maintain a clear idea of what types can be "upcast" to +others, possibly indirectly (e.g. indirect A->B->C is implemented but +direct A->C not). If the implementations of ``__array_ufunc__`` follow a +coherent type casting hierarchy, it can be used to understand results of +operations. + +Type casting can be expressed as a `graph `__ +defined as follows: + + For each ``__array_ufunc__`` method, draw directed edges from each + possible input type to each possible output type. + + That is, in each case where ``y = x.__array_ufunc__(a, b, c, ...)`` + does something else than returning ``NotImplemented`` or raising an error, + draw edges ``type(a) -> type(y)``, ``type(b) -> type(y)``, ... -In particular, it is useful to maintain a clear idea of what types can -be upcast to others, possibly indirectly (i.e. A->B->C is implemented -but direct A->C not). Moreover, one should make sure the implementations of -``__array_ufunc__``, which implicitly define the type casting hierarchy, -don't contradict this. +If the resulting graph is *acyclic*, it defines a coherent type casting +hierarchy (unambiguous partial ordering between types). In this case, +operations involving multiple types generally predictably produce result +of the "highest" type, or raise a :exc:`TypeError`. See examples at the +end of this section. -It is useful to think of the typecasting hierarchy as a graph (see below -for example graphs that work and that fail because of cyclic -dependencies) in which, for any given class A, all other classes that -define ``__array_ufunc__`` must belong to exactly one of three groups -(making this an directed acyclic graph): +If the graph has cycles, the ``__array_ufunc__`` type casting is not +well-defined, and things such as ``type(multiply(a, b)) != +type(multiply(b, a))`` or ``type(add(a, add(b, c))) != type(add(add(a, +b), c))`` are not excluded (and then probably always possible). -- *Above A*: their ``__array_ufunc__`` can handle class A or some - member of the "above A" classes. In other words, these are the types - that A can be (indirectly) upcast to in ufuncs. +If the type casting hierarchy is well defined, for each class A, all +other classes that define ``__array_ufunc__`` belong to exactly one of +three groups: -- *Below A*: they can be handled by the ``__array_ufunc__`` of class A - or the ``__array_ufunc__`` of some member of the "below A" classes. In - other words, these are the types that can be (indirectly) upcast to A - in ufuncs. +- *Above A*: the types that A can be (indirectly) upcast to in ufuncs. + +- *Below A*: the types that can be (indirectly) upcast to A in ufuncs. - *Incompatible*: neither above nor below A; types for which no - (indirect) upcasting is possible. Neither can handle the other. - -Given this grouping, to ensure that expressions involving ufuncs either -raise a :exc:`TypeError`, or have a result type that is independent of -what ufuncs were called, what order they were called in, and what order -their arguments were in, the above implies that ``__array_ufunc__`` for -type A should: - -- Return an object of type A if all other arguments are of types below A. - -- Return :obj:`NotImplemented` if any argument has a type that is above - A or with which it is incompatible. - -With the above, one can convert relations between types to edges in a -`graph`_ by defining "can -handle" as follows: if for instances ``a`` and ``b`` of types A and B, -``a.__array_ufunc__(..., b, ...)`` returns a result other than -:obj:`NotImplemented` (and does not raise an error), then a can handle -b and B->A is an edge of the graph. - -Note that there are, as always, exceptions. For instance, for a -quantity class, the results of most ufuncs should be quantities, but -this is not the case for comparison operators. For those, a quantity -class would return a plain array. - -Note also that the legacy behaviour of numpy ufunc is to try to convert + (indirect) upcasting is possible. + +Note that the legacy behaviour of numpy ufuncs is to try to convert unknown objects to :class:`ndarray` via :func:`np.asarray`. This is -equivalent to placing :class:`ndarray` at the very top of the graph, and -is thus a consistent type hierarchy (although one that causes the -problems that motivate this NEP...). By instead letting -:class:`ndarray` return `NotImplemented` if any argument defines -``__array_ufunc__``, we provide the option for other classes to have -:class:`ndarray` at the bottom of the type hierarchy. +equivalent to placing :class:`ndarray` above these objects in the graph. +Since we above defined :class:`ndarray` to return `NotImplemented` for +classes with custom ``__array_ufunc__``, this puts :class:`ndarray` +below such classes in the type hierarchy, allowing the operations to be +overridden. + +In view of the above, binary ufuncs describing transitive operations +should aim to define a well-defined casting hierarchy. This is likely +also a sensible approach to all ufuncs --- exceptions to this should +consider carefully if any surprising behavior results. .. admonition:: Example @@ -262,19 +260,20 @@ problems that motivate this NEP...). By instead letting digraph array_ufuncs { rankdir=BT; - A -> C; - B -> C; - D -> B; - ndarray -> A; - ndarray -> B; + A -> C [label="C"]; + B -> C [label="C"]; + D -> B [label="B"]; + ndarray -> C [label="A"]; + ndarray -> B [label="B"]; } - The ``__array_ufunc__`` of type A can handle ndarrays, B can handle - ndarray and D, and C can handle A and B but not ndarrays or D. The + The ``__array_ufunc__`` of type A can handle ndarrays returning C, + B can handle ndarray and D returning B, and C can handle A and B returning C, + but not ndarrays or D. The result is a directed acyclic graph, and defines a type casting - hierarchy, with relations ``C > A > ndarray``, ``C > B > ndarray``, - ``C > B > D``. The type B is incompatible relative to A and vice - versa, and A and ndarray are incompatible relative to D. Ufunc + hierarchy, with relations ``C > A``, ``C > ndarray``, ``C > B > ndarray``, + ``C > B > D``. The type A is incompatible with B, D, ndarray, + and D is incompatible with A and ndarray. Ufunc expressions involving these classes should produce results of the highest type involved or raise a :exc:`TypeError`. @@ -286,8 +285,8 @@ problems that motivate this NEP...). By instead letting digraph array_ufuncs { rankdir=BT; - A -> B; - B -> A; + A -> B [label="B"]; + B -> A [label="A"]; } @@ -303,9 +302,9 @@ problems that motivate this NEP...). By instead letting digraph array_ufuncs { rankdir=BT; - A -> B; - B -> C; - C -> A; + A -> B [label="B"]; + B -> C [label="C"]; + C -> A [label="A"]; } @@ -324,10 +323,18 @@ type casting hierarchy. The recommendation is that an `NotImplemented` unless the inputs are instances of the same class or superclasses. This guarantees that in the type casting hierarchy, superclasses are below, subclasses above, and other classes are -incompatible (sadly, the terminology for graphs and classes has reversed -vertical sense). Exceptions to this need to check they respect the +incompatible. Exceptions to this need to check they respect the implicit type casting hierarchy. +.. note:: + + Note that type casting hierarchy and class hierarchy are here defined + to go the "opposite" directions. It would in principle also be + consistent to have ``__array_ufunc__`` handle also instances of + subclasses. In this case, the "subclasses first" dispatch rule would + ensure a relatively similar outcome. However, the behavior is then less + explicitly specified. + Subclasses can be easily constructed if methods consistently use :func:`super` to pass through the class hierarchy [7]_. To support this, :class:`ndarray` has its own ``__array_ufunc__`` method, From ff628f1697538ba03c9fc02409b96896fa56e9f6 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 9 Apr 2017 11:45:46 -0600 Subject: [PATCH 34/43] BUG: Add back removed elision code. Code inadvertantly removed in rebasing old PR on new work. --- numpy/core/src/multiarray/number.c | 72 +++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index f846fb318990..ad1d43178c60 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -283,21 +283,36 @@ array_inplace_right_shift(PyArrayObject *m1, PyObject *m2); static PyObject * array_add(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_add, array_add); + if (try_binary_elide(m1, m2, &array_inplace_add, &res, 1)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.add); } static PyObject * array_subtract(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_subtract, array_subtract); + if (try_binary_elide(m1, m2, &array_inplace_subtract, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.subtract); } static PyObject * array_multiply(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_multiply, array_multiply); + if (try_binary_elide(m1, m2, &array_inplace_multiply, &res, 1)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.multiply); } @@ -305,7 +320,12 @@ array_multiply(PyArrayObject *m1, PyObject *m2) static PyObject * array_divide(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_divide, array_divide); + if (try_binary_elide(m1, m2, &array_inplace_divide, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.divide); } #endif @@ -343,11 +363,12 @@ array_inplace_matrix_multiply(PyArrayObject *m1, PyObject *m2) } #endif -/* Determine if object is a scalar and if so, convert the object - * to a double and place it in the out_exponent argument - * and return the "scalar kind" as a result. If the object is - * not a scalar (or if there are other error conditions) - * return NPY_NOSCALAR, and out_exponent is undefined. +/* + * Determine if object is a scalar and if so, convert the object + * to a double and place it in the out_exponent argument + * and return the "scalar kind" as a result. If the object is + * not a scalar (or if there are other error conditions) + * return NPY_NOSCALAR, and out_exponent is undefined. */ static NPY_SCALARKIND is_scalar_with_conversion(PyObject *o2, double* out_exponent) @@ -458,7 +479,8 @@ fast_scalar_power(PyArrayObject *a1, PyObject *o2, int inplace) if (inplace || can_elide_temp_unary(a1)) { return PyArray_GenericInplaceUnaryFunction(a1, fastop); - } else { + } + else { return PyArray_GenericUnaryFunction(a1, fastop); } } @@ -546,35 +568,60 @@ array_invert(PyArrayObject *m1) static PyObject * array_left_shift(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_lshift, array_left_shift); + if (try_binary_elide(m1, m2, &array_inplace_left_shift, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.left_shift); } static PyObject * array_right_shift(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_rshift, array_right_shift); + if (try_binary_elide(m1, m2, &array_inplace_right_shift, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.right_shift); } static PyObject * array_bitwise_and(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_and, array_bitwise_and); + if (try_binary_elide(m1, m2, &array_inplace_bitwise_and, &res, 1)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_and); } static PyObject * array_bitwise_or(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_or, array_bitwise_or); + if (try_binary_elide(m1, m2, &array_inplace_bitwise_or, &res, 1)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_or); } static PyObject * array_bitwise_xor(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_xor, array_bitwise_xor); + if (try_binary_elide(m1, m2, &array_inplace_bitwise_xor, &res, 1)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.bitwise_xor); } @@ -655,14 +702,26 @@ array_inplace_bitwise_xor(PyArrayObject *m1, PyObject *m2) static PyObject * array_floor_divide(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_floor_divide, array_floor_divide); + if (try_binary_elide(m1, m2, &array_inplace_floor_divide, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.floor_divide); } static PyObject * array_true_divide(PyArrayObject *m1, PyObject *m2) { + PyObject *res; + BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_true_divide, array_true_divide); + if (PyArray_CheckExact(m1) && + (PyArray_ISFLOAT(m1) || PyArray_ISCOMPLEX(m1)) && + try_binary_elide(m1, m2, &array_inplace_true_divide, &res, 0)) { + return res; + } return PyArray_GenericBinaryFunction(m1, m2, n_ops.true_divide); } @@ -708,6 +767,7 @@ static PyObject * array_divmod(PyArrayObject *op1, PyObject *op2) { PyObject *divp, *modp, *result; + BINOP_GIVE_UP_IF_NEEDED(op1, op2, nb_divmod, array_divmod); divp = array_floor_divide(op1, op2); From 1fc6e633e39cd837cd296104022558b2b880ab29 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Mon, 10 Apr 2017 15:00:00 -0400 Subject: [PATCH 35/43] DOC,TST: clarify example of ndarray subclass using __array_ufunc__ Also add a few more tests of the same example for good measure. --- numpy/core/tests/test_umath.py | 33 ++++++++++++++++++++++++++------- numpy/doc/subclassing.py | 13 ++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 454f2702035f..eee0324172ad 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1926,7 +1926,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): else: args.append(input_) - outputs = kwargs.pop('out', []) + outputs = kwargs.pop('out', None) out_no = [] if outputs: out_args = [] @@ -1937,6 +1937,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): else: out_args.append(output) kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout info = {} if in_no: @@ -1946,20 +1948,27 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): results = super(A, self).__array_ufunc__(ufunc, method, *args, **kwargs) - if not isinstance(results, tuple): - if not isinstance(results, np.ndarray): - return results + if results is NotImplemented: + return NotImplemented + + if ufunc.nout == 1: results = (results,) - if outputs == []: - outputs = [None] * len(results) - results = tuple(result.view(A) if output is None else output + results = tuple((np.asarray(result).view(A) + if output is None else output) for result, output in zip(results, outputs)) if isinstance(results[0], A): results[0].info = info return results[0] if len(results) == 1 else results + class B(object): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + if any(isinstance(input_, A) for input_ in inputs): + return "A!" + else: + return NotImplemented + d = np.arange(5.) a = np.arange(5.).view(A) # 1 input, 1 output @@ -1994,6 +2003,16 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): c = np.add(a, b, out=a) assert_(c is a) assert_equal(c.info, {'inputs': [0, 1], 'outputs': [0]}) + # some tests with a non-ndarray subclass + a = np.arange(5.) + b = B() + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_raises(TypeError, np.add, a, b) + a = a.view(A) + assert_(a.__array_ufunc__(np.add, '__call__', a, b) is NotImplemented) + assert_(b.__array_ufunc__(np.add, '__call__', a, b) == "A!") + assert_(np.add(a, b) == "A!") class TestChoose(TestCase): diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 7877432b54b3..410fc58bd408 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -465,7 +465,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): outputs = kwargs.pop('out', None) out_no = [] - if outputs is not None: + if outputs: out_args = [] for j, output in enumerate(outputs): if isinstance(output, A): @@ -474,6 +474,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): else: out_args.append(output) kwargs['out'] = tuple(out_args) + else: + outputs = (None,) * ufunc.nout info = {} if in_no: @@ -486,14 +488,11 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): if results is NotImplemented: return NotImplemented - if not isinstance(results, tuple): - if not isinstance(results, np.ndarray): - return results + if ufunc.nout == 1: results = (results,) - if outputs is None: - outputs = [None] * len(results) - results = tuple(result.view(A) if output is None else output + results = tuple((np.asarray(result).view(A) + if output is None else output) for result, output in zip(results, outputs)) if isinstance(results[0], A): results[0].info = info From a43174390de55b2496a9e9aed84cf424229e817a Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 12 Apr 2017 10:32:19 +0100 Subject: [PATCH 36/43] BUG: Support nout == 0 and at method --- numpy/core/tests/test_umath.py | 9 +++++++-- numpy/doc/subclassing.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index eee0324172ad..97e893c681df 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1951,13 +1951,16 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): if results is NotImplemented: return NotImplemented + if method == 'at': + return + if ufunc.nout == 1: results = (results,) results = tuple((np.asarray(result).view(A) if output is None else output) for result, output in zip(results, outputs)) - if isinstance(results[0], A): + if results and isinstance(results[0], A): results[0].info = info return results[0] if len(results) == 1 else results @@ -1970,8 +1973,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return NotImplemented d = np.arange(5.) - a = np.arange(5.).view(A) # 1 input, 1 output + a = np.arange(5.).view(A) b = np.sin(a) check = np.sin(d) assert_(np.all(check == b)) @@ -1984,6 +1987,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): b = np.sin(a, out=a) assert_(np.all(check == b)) assert_equal(b.info, {'inputs': [0], 'outputs': [0]}) + # 1 input, 2 outputs a = np.arange(5.).view(A) b1, b2 = np.modf(a) @@ -1997,6 +2001,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_(c1 is a) assert_(c2 is b) assert_equal(c1.info, {'inputs': [0], 'outputs': [0, 1]}) + # 2 input, 1 output a = np.arange(5.).view(A) b = np.arange(5.).view(A) diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 410fc58bd408..36d8ff97d285 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -488,13 +488,16 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): if results is NotImplemented: return NotImplemented + if method == 'at': + return + if ufunc.nout == 1: results = (results,) results = tuple((np.asarray(result).view(A) if output is None else output) for result, output in zip(results, outputs)) - if isinstance(results[0], A): + if results and isinstance(results[0], A): results[0].info = info return results[0] if len(results) == 1 else results From 1e460b74bac7da0d9029b1fd414213f00bb66c9f Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Wed, 12 Apr 2017 10:50:38 -0400 Subject: [PATCH 37/43] DOC,MAINT: small corrections to NEP following Stephan's comments. --- doc/neps/ufunc-overrides.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 1eaefbf5df5b..239c4a480bd8 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -169,7 +169,7 @@ The function dispatch proceeds as follows: - If one of the input or output arguments implements ``__array_ufunc__``, it is executed instead of the ufunc. -- If more than one of the input arguments implements ``__array_ufunc__``, +- If more than one of the arguments implements ``__array_ufunc__``, they are tried in the following order: subclasses before superclasses, inputs before outputs, otherwise left to right. @@ -343,7 +343,7 @@ equivalent to:: def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # Cannot handle items that have __array_ufunc__ (other than our own). outputs = kwargs.get('out', ()) - for item in inputs + outputs): + for item in inputs + outputs: if (hasattr(item, '__array_ufunc__') and type(item).__array_ufunc__ is not ndarray.__array_ufunc__): return NotImplemented @@ -372,8 +372,9 @@ the superclass implementation using :func:`super` until the ufunc is actually done, and then do possible adjustments of the outputs. In general, custom implementations of `__array_ufunc__` should avoid -nested dispatch cycles. However, for some subclasses, it may be better -to use ``getattr(ufunc, method)(*items, **kwargs)``. For instance, for a +nested dispatch cycles, where one not just calls the ufunc via +``getattr(ufunc, method)(*items, **kwargs)``, but catches possible +exceptions, etc. As always, there may be exceptions. For instance, for a class like :class:`MaskedArray`, which only cares that whatever it contains is an :class:`ndarray` subclass, a reimplementation with ``__array_ufunc__`` may well be more easily done by directly applying @@ -385,8 +386,8 @@ the implementation would be something like:: def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # for simplicity, outputs are ignored here. - unmasked_items = [item.data if isinstance(item, MaskedArray) - else item] + unmasked_items = tuple((item.data if isinstance(item, MaskedArray) + else item) for item in inputs) try: unmasked_result = getattr(ufunc, method)(*unmasked_items, **kwargs) except TypeError: @@ -549,7 +550,7 @@ with ``mine`` and will thus return :obj:`NotImplemented`. Then, the ufunc turns to ``mine.__array_ufunc__``. But this is :obj:`None`, equivalent to returning :obj:`NotImplemented`, so a :exc:`TypeError` is raised. In option (2), we pass directly to ``arr.__array_ufunc__``, -which will return :obj:`NotImplemted`, which we catch. +which will return :obj:`NotImplemented`, which we catch. .. note :: the reason for not allowing in-place operations to return :obj:`NotImplemented` is that these cannot generically be replaced by From 02600d38f3b2e70c3cd07770f93c3bac5255c8a6 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 21 Apr 2017 09:35:45 -0700 Subject: [PATCH 38/43] ENH: Add NDArrayOperatorsMixin mixin class. This mixin class provides an easy way to implement arithmetic operators that defer to __array_ufunc__ like numpy.ndarray in non-ndarray subclasses. --- doc/neps/ufunc-overrides.rst | 14 +- doc/source/reference/arrays.classes.rst | 4 +- doc/source/reference/routines.other.rst | 7 + numpy/lib/__init__.py | 2 + numpy/lib/mixins.py | 167 +++++++++++++++++++++ numpy/lib/tests/test_mixins.py | 189 ++++++++++++++++++++++++ 6 files changed, 375 insertions(+), 8 deletions(-) create mode 100644 numpy/lib/mixins.py create mode 100644 numpy/lib/tests/test_mixins.py diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst index 239c4a480bd8..0c2c1ff03016 100644 --- a/doc/neps/ufunc-overrides.rst +++ b/doc/neps/ufunc-overrides.rst @@ -1,3 +1,5 @@ +.. _neps.ufunc-overrides: + ================================= A Mechanism for Overriding Ufuncs ================================= @@ -5,7 +7,7 @@ A Mechanism for Overriding Ufuncs .. currentmodule:: numpy :Author: Blake Griffith -:Contact: blake.g@utexas.edu +:Contact: blake.g@utexas.edu :Date: 2013-07-10 :Author: Pauli Virtanen @@ -94,7 +96,7 @@ Take this example of ufuncs interoperability with sparse matrices.:: Out[4]: matrix([[16, 0, 8], [ 8, 1, 5], [ 4, 1, 4]], dtype=int64) - + In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! Out[5]: NotImplemted @@ -147,11 +149,11 @@ signature is:: Here: -- *ufunc* is the ufunc object that was called. +- *ufunc* is the ufunc object that was called. - *method* is a string indicating how the Ufunc was called, either ``"__call__"`` to indicate it was called directly, or one of its :ref:`methods`: ``"reduce"``, ``"accumulate"``, - ``"reduceat"``, ``"outer"``, or ``"at"``. + ``"reduceat"``, ``"outer"``, or ``"at"``. - *inputs* is a tuple of the input arguments to the ``ufunc`` - *kwargs* contains any optional or keyword arguments passed to the function. This includes any ``out`` arguments, which are always @@ -578,8 +580,8 @@ make more sense to ask classes like ``MyObject`` to implement a full ``__array_ufunc__`` [6]_. In the end, allowing classes to opt out was preferred, and the above reasoning led us to agree on a similar implementation for :class:`ndarray` itself. To help implement array-like -classes, we will provide a mixin class that provides overrides for all -binary operators. +classes, the mixin class :class:`~numpy.lib.mixins.NDArrayOperatorsMixin` +provides overrides for all binary operators with corresponding ufuncs. Future extensions to other functions diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 219f9ca6490f..e451fea68a31 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -127,8 +127,8 @@ NumPy provides several hooks that classes can customize: - If you are not a subclass of :class:`ndarray`, we recommend your class define special methods like ``__add__`` and ``__lt__`` that - delegate to ufuncs just like ndarray does. We hope to provide a - helper mixin class for this. + delegate to ufuncs just like ndarray does. An easy way to do this + is to subclass from :class:`~numpy.lib.mixins.NDArrayOperatorsMixin`. - If you subclass :class:`ndarray`, we strongly recommend that you avoid confusion by neither setting :func:`__array_ufunc__` to :obj:`None`, which makes no sense for an array subclass, nor by diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index 5a5f2b81855d..45b9ac3d94da 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -30,6 +30,13 @@ Memory ranges shares_memory may_share_memory +Array mixins +------------ +.. autosummary:: + :toctree: generated/ + + lib.mixins.NDArrayOperatorsMixin + NumPy version comparison ------------------------ .. autosummary:: diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index 1d65db55e18e..4cdb76b20ef9 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -8,6 +8,7 @@ from .type_check import * from .index_tricks import * from .function_base import * +from .mixins import * from .nanfunctions import * from .shape_base import * from .stride_tricks import * @@ -29,6 +30,7 @@ __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ +__all__ += mixins.__all__ __all__ += shape_base.__all__ __all__ += stride_tricks.__all__ __all__ += twodim_base.__all__ diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py new file mode 100644 index 000000000000..877a11039b4b --- /dev/null +++ b/numpy/lib/mixins.py @@ -0,0 +1,167 @@ +"""Mixin classes for custom array types that don't inherit from ndarray.""" +from __future__ import division, absolute_import, print_function + +import sys + +from numpy.core import umath as um + +# None of this module should be exposed in top-level NumPy module. +__all__ = [] + + +def _binary_method(ufunc): + def func(self, other): + try: + if other.__array_ufunc__ is None: + return NotImplemented + except AttributeError: + pass + return self.__array_ufunc__(ufunc, '__call__', self, other) + return func + + +def _reflected_binary_method(ufunc): + def func(self, other): + try: + if other.__array_ufunc__ is None: + return NotImplemented + except AttributeError: + pass + return self.__array_ufunc__(ufunc, '__call__', other, self) + return func + + +def _inplace_binary_method(ufunc): + def func(self, other): + result = self.__array_ufunc__( + ufunc, '__call__', self, other, out=(self,)) + if result is NotImplemented: + raise TypeError('unsupported operand types for in-place ' + 'arithmetic: %s and %s' + % (type(self).__name__, type(other).__name__)) + return result + return func + + +def _numeric_methods(ufunc): + return (_binary_method(ufunc), + _reflected_binary_method(ufunc), + _inplace_binary_method(ufunc)) + + +def _unary_method(ufunc): + def func(self): + return self.__array_ufunc__(ufunc, '__call__', self) + return func + + +class NDArrayOperatorsMixin(object): + """Mixin defining all operator special methods using __array_ufunc__. + + This class implements the special methods for almost all of Python's + builtin operators defined in the `operator` module, including comparisons + (``==``, ``>``, etc.) and arithmetic (``+``, ``*``, ``-``, etc.), by + deferring to the ``__array_ufunc__`` method, which subclasses must + implement. + + This class does not yet implement the special operators corresponding + to ``divmod``, unary ``+`` or ``matmul`` (``@``), because these operation + do not yet have corresponding NumPy ufuncs. + + It is useful for writing classes that do not inherit from `numpy.ndarray`, + but that should support arithmetic and numpy universal functions like + arrays as described in :ref:`A Mechanism for Overriding Ufuncs + `. + + As an trivial example, consider this implementation of an ``ArrayLike`` + class that simply wraps a NumPy array and ensures that the result of any + arithmetic operation is also an ``ArrayLike`` object:: + + class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES + # and superclass instances of this type + if not (isinstance(x, self._HANDLED_TYPES) or + isinstance(self, type(x))): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values + inputs = tuple(x.value if isinstance(self, type(x)) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(self, type(x)) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + In interactions between ``ArrayLike`` objects and numbers or numpy arrays, + the result is always another ``ArrayLike``: + + >>> x = ArrayLike([1, 2, 3]) + >>> x - 1 + ArrayLike(array([0, 1, 2])) + >>> 1 - x + ArrayLike(array([ 0, -1, -2])) + >>> np.arange(3) - x + ArrayLike(array([-1, -1, -1])) + >>> x - np.arange(3) + ArrayLike(array([1, 1, 1])) + + Note that unlike ``numpy.ndarray``, ``ArrayLike`` does not allow operations + with arbitrary, unrecognized types. This ensures that interactions with + ArrayLike preserve a well-defined casting hierarchy. + """ + + # comparisons don't have reflected and in-place versions + __lt__ = _binary_method(um.less) + __le__ = _binary_method(um.less_equal) + __eq__ = _binary_method(um.equal) + __ne__ = _binary_method(um.not_equal) + __gt__ = _binary_method(um.greater) + __ge__ = _binary_method(um.greater_equal) + + # numeric methods + __add__, __radd__, __iadd__ = _numeric_methods(um.add) + __sub__, __rsub__, __isub__ = _numeric_methods(um.subtract) + __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply) + if sys.version_info.major < 3: + # Python 3 uses only __truediv__ and __floordiv__ + __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide) + __truediv__, __rtruediv__, __itruediv__ = _numeric_methods(um.true_divide) + __floordiv__, __rfloordiv__, __ifloordiv__ = _numeric_methods( + um.floor_divide) + __mod__, __rmod__, __imod__ = _numeric_methods(um.mod) + # TODO: handle the optional third argument for __pow__? + __pow__, __rpow__, __ipow__ = _numeric_methods(um.power) + __lshift__, __rlshift__, __ilshift__ = _numeric_methods(um.left_shift) + __rshift__, __rrshift__, __irshift__ = _numeric_methods(um.right_shift) + __and__, __rand__, __iand__ = _numeric_methods(um.bitwise_and) + __xor__, __rxor__, __ixor__ = _numeric_methods(um.bitwise_xor) + __or__, __ror__, __ior__ = _numeric_methods(um.bitwise_or) + + # unary methods + __neg__ = _unary_method(um.negative) + __abs__ = _unary_method(um.absolute) + __invert__ = _unary_method(um.invert) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py new file mode 100644 index 000000000000..f45a3c6617e8 --- /dev/null +++ b/numpy/lib/tests/test_mixins.py @@ -0,0 +1,189 @@ +from __future__ import division, absolute_import, print_function + +import numbers +import operator +import sys + +import numpy as np +from numpy.testing import ( + TestCase, run_module_suite, assert_, assert_equal, assert_raises) + + +PY2 = sys.version_info.major < 3 + + +# NOTE: This class should be kept as an exact copy of the example from the +# docstring for NDArrayOperatorsMixin. + +class ArrayLike(np.lib.mixins.NDArrayOperatorsMixin): + def __init__(self, value): + self.value = np.asarray(value) + + # One might also consider adding the built-in list type to this + # list, to support operations like np.add(array_like, list) + _HANDLED_TYPES = (np.ndarray, numbers.Number) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + out = kwargs.get('out', ()) + for x in inputs + out: + # Only support operations with instances of _HANDLED_TYPES + # and superclass instances of this type + if not (isinstance(x, self._HANDLED_TYPES) or + isinstance(self, type(x))): + return NotImplemented + + # Defer to the implementation of the ufunc on unwrapped values + inputs = tuple(x.value if isinstance(self, type(x)) else x + for x in inputs) + if out: + kwargs['out'] = tuple( + x.value if isinstance(self, type(x)) else x + for x in out) + result = getattr(ufunc, method)(*inputs, **kwargs) + + if type(result) is tuple: + # multiple return values + return tuple(type(self)(x) for x in result) + elif method == 'at': + # no return value + return None + else: + # one return value + return type(self)(result) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.value) + + +def _assert_equal_type_and_value(result, expected, err_msg=None): + assert_equal(type(result), type(expected), err_msg=err_msg) + assert_equal(result.value, expected.value, err_msg=err_msg) + assert_equal(getattr(result.value, 'dtype', None), + getattr(expected.value, 'dtype', None), err_msg=err_msg) + + +class TestNDArrayOperatorsMixin(TestCase): + + def test_array_like_add(self): + + def check(result): + _assert_equal_type_and_value(result, ArrayLike(0)) + + check(ArrayLike(0) + 0) + check(0 + ArrayLike(0)) + + check(ArrayLike(0) + np.array(0)) + check(np.array(0) + ArrayLike(0)) + + check(ArrayLike(np.array(0)) + 0) + check(0 + ArrayLike(np.array(0))) + + check(ArrayLike(np.array(0)) + np.array(0)) + check(np.array(0) + ArrayLike(np.array(0))) + + def test_inplace(self): + array_like = ArrayLike(np.array([0])) + array_like += 1 + _assert_equal_type_and_value(array_like, ArrayLike(np.array([1]))) + + array = np.array([0]) + array += ArrayLike(1) + _assert_equal_type_and_value(array, ArrayLike(np.array([1]))) + + def test_opt_out(self): + + class OptOut(object): + """Object that opts out of __array_ufunc__.""" + __array_ufunc__ = None + + def __add__(self, other): + return self + + def __radd__(self, other): + return self + + array_like = ArrayLike(1) + opt_out = OptOut() + + # supported operations + assert_(array_like + opt_out is opt_out) + assert_(opt_out + array_like is opt_out) + + # not supported + with assert_raises(TypeError): + # don't use the Python default, array_like = array_like + opt_out + array_like += opt_out + with assert_raises(TypeError): + array_like - opt_out + with assert_raises(TypeError): + opt_out - array_like + + def test_subclass(self): + + class SubArrayLike(ArrayLike): + """Should take precedence over ArrayLike.""" + + x = ArrayLike(0) + y = SubArrayLike(1) + _assert_equal_type_and_value(x + y, y) + _assert_equal_type_and_value(y + x, y) + + def test_unary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + for op in [operator.neg, + # pos is not yet implemented + abs, + operator.invert]: + _assert_equal_type_and_value(op(array_like), ArrayLike(op(array))) + + def test_binary_methods(self): + array = np.array([-1, 0, 1, 2]) + array_like = ArrayLike(array) + operators = [ + operator.lt, + operator.le, + operator.eq, + operator.ne, + operator.gt, + operator.ge, + operator.add, + operator.sub, + operator.mul, + operator.truediv, + operator.floordiv, + # TODO: test div on Python 2, only + operator.mod, + # divmod is not yet implemented + pow, + operator.lshift, + operator.rshift, + operator.and_, + operator.xor, + operator.or_, + ] + for op in operators: + expected = ArrayLike(op(array, 1)) + actual = op(array_like, 1) + err_msg = 'failed for operator {}'.format(op) + _assert_equal_type_and_value(expected, actual, err_msg=err_msg) + + def test_ufunc_at(self): + array = ArrayLike(np.array([1, 2, 3, 4])) + assert_(np.negative.at(array, np.array([0, 1])) is None) + _assert_equal_type_and_value(array, ArrayLike([-1, -2, 3, 4])) + + def test_ufunc_two_outputs(self): + def check(result): + assert_(type(result) is tuple) + assert_equal(len(result), 2) + mantissa, exponent = np.frexp(2 ** -3) + _assert_equal_type_and_value(result[0], ArrayLike(mantissa)) + _assert_equal_type_and_value(result[1], ArrayLike(exponent)) + + check(np.frexp(ArrayLike(2 ** -3))) + check(np.frexp(ArrayLike(np.array(2 ** -3)))) + + +if __name__ == "__main__": + run_module_suite() From d3ff023d45768062686f0bc9555360e90967f07f Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Fri, 21 Apr 2017 17:35:48 -0400 Subject: [PATCH 39/43] DOC: clarify recommendations for subclasses, deprecations. --- doc/release/1.13.0-notes.rst | 2 +- doc/source/reference/arrays.classes.rst | 31 +++++++++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/doc/release/1.13.0-notes.rst b/doc/release/1.13.0-notes.rst index dde2f1fd86c2..b436a96e6a76 100644 --- a/doc/release/1.13.0-notes.rst +++ b/doc/release/1.13.0-notes.rst @@ -8,7 +8,7 @@ Highlights ========== * Operations like ``a + b + c`` will reuse temporaries on some platforms, - resulting less memory use and faster execution. + resulting in less memory use and faster execution. * Inplace operations check if inputs overlap outputs and create temporaries to avoid problems. * Improved ability for classes to override default ufunc behavior. See diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index e451fea68a31..25105001ce68 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -120,8 +120,9 @@ NumPy provides several hooks that classes can customize: never returns :obj:`NotImplemented`. Hence, ``arr += obj`` would always lead to a :exc:`TypeError`. This is because for arrays in-place operations cannot generically be replaced by a simple reverse operation. (For - instance, by default, ``arr[:] += obj`` would be translated to ``arr[:] = - arr[:] + obj``, which would likely be wrong.) + instance, by default, ``arr += obj`` would be translated to ``arr = + arr + obj``, i.e., ``arr`` would be replaced, contrary to what is expected + for in-place array operations.) .. note:: If you define ``__array_ufunc__``: @@ -129,12 +130,15 @@ NumPy provides several hooks that classes can customize: class define special methods like ``__add__`` and ``__lt__`` that delegate to ufuncs just like ndarray does. An easy way to do this is to subclass from :class:`~numpy.lib.mixins.NDArrayOperatorsMixin`. - - If you subclass :class:`ndarray`, we strongly recommend that you - avoid confusion by neither setting :func:`__array_ufunc__` to - :obj:`None`, which makes no sense for an array subclass, nor by - defining it and also defining reverse methods, which methods will - be called by ``CPython`` in preference over the :class:`ndarray` - forward methods. + - If you subclass :class:`ndarray`, we recommend that you put all your + override logic in ``__array_ufunc__`` and not also override special + methods. This ensures the class hierarchy is determined in only one + place rather than separately by the ufunc machinery and by the binary + operation rules (which gives preference to special methods of + subclasses; the alternative way to enforce a one-place only hierarchy, + of setting :func:`__array_ufunc__` to :obj:`None`, would seem very + unexpected and thus confusing, as then the subclass would not work at + all with ufuncs). - :class:`ndarray` defines its own :func:`__array_ufunc__`, which, evaluates the ufunc if no arguments have overrides, and returns :obj:`NotImplemented` otherwise. This may be useful for subclasses @@ -172,8 +176,8 @@ NumPy provides several hooks that classes can customize: the subclass and update metadata before returning the array to the ufunc for computation. - .. note:: It is hoped to eventually deprecate this method in favour of - :func:`__array_ufunc__`. + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. .. py:method:: class.__array_wrap__(array, context=None) @@ -187,6 +191,9 @@ NumPy provides several hooks that classes can customize: into an instance of the subclass and update metadata before returning the array to the user. + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. + .. py:attribute:: class.__array_priority__ The value of this attribute is used to determine what type of @@ -194,8 +201,8 @@ NumPy provides several hooks that classes can customize: possibility for the Python type of the returned object. Subclasses inherit a default value of 0.0 for this attribute. - .. note:: It is hoped to eventually deprecate this attribute in favour - of :func:`__array_ufunc__`. + .. note:: For ufuncs, it is hoped to eventually deprecate this method in + favour of :func:`__array_ufunc__`. .. py:method:: class.__array__([dtype]) From b9359f1d9fede0d4ecc08e868e2b0dcb85dbd7e2 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Fri, 21 Apr 2017 17:35:20 -0400 Subject: [PATCH 40/43] MAINT: remove unnecessary checks, wrong code for 'outer', cleanup. --- numpy/core/src/multiarray/methods.c | 8 -- numpy/core/src/umath/override.c | 165 ++++++++++++++++++++++------ numpy/core/src/umath/ufunc_object.c | 21 ++-- numpy/core/tests/test_umath.py | 13 +++ 4 files changed, 153 insertions(+), 54 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 26f98e006a35..946dc542f090 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1030,15 +1030,7 @@ array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) } ufunc = PyTuple_GET_ITEM(args, 0); - if (ufunc == NULL) { - goto cleanup; - } - method_name = PyTuple_GET_ITEM(args, 1); - if (method_name == NULL) { - goto cleanup; - } - /* * TODO(?): call into UFunc code at a later point, since here arguments are * already normalized and we do not have to look for __array_ufunc__ again. diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index 6cd4bee14b7e..d059622c5edb 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -14,11 +14,35 @@ * generalized ufuncs, and by PyUFunc_GenericReduction for the other methods. * It would be good to unify (see gh-8892). */ + +/* + * ufunc() and ufunc.outer() accept 'sig' or 'signature'; + * normalize to 'signature' + */ +static int +normalize_signature_keyword(PyObject *normal_kwds) +{ + PyObject* obj = PyDict_GetItemString(normal_kwds, "sig"); + if (obj != NULL) { + if (PyDict_GetItemString(normal_kwds, "signature")) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + return -1; + } + Py_INCREF(obj); + PyDict_SetItemString(normal_kwds, "signature", obj); + PyDict_DelItemString(normal_kwds, "sig"); + } + return 0; +} + static int normalize___call___args(PyUFuncObject *ufunc, PyObject *args, PyObject **normal_args, PyObject **normal_kwds) { - /* ufunc.__call__(*args, **kwds) */ + /* + * ufunc.__call__(*args, **kwds) + */ int i; int not_all_none; int nin = ufunc->nin; @@ -28,27 +52,16 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, if (nargs < nin) { PyErr_Format(PyExc_TypeError, - "required input argument (pos %d) not found", nin); + "ufunc() missing %d of %d required positional argument(s)", + nin - nargs, nin); return -1; } if (nargs > nin+nout) { PyErr_Format(PyExc_TypeError, - "ufunc takes at most %d arguments (%d given)", - nin+nout, nargs); + "ufunc() takes from %d to %d arguments but %d were given", + nin, nin+nout, nargs); return -1; } - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - obj = PyDict_GetItemString(*normal_kwds, "sig"); - if (obj != NULL) { - if (PyDict_GetItemString(*normal_kwds, "signature")) { - PyErr_SetString(PyExc_TypeError, - "cannot specify both 'sig' and 'signature'"); - return -1; - } - Py_INCREF(obj); - PyDict_SetItemString(*normal_kwds, "signature", obj); - PyDict_DelItemString(*normal_kwds, "sig"); - } *normal_args = PyTuple_GetSlice(args, 0, nin); if (*normal_args == NULL) { @@ -63,15 +76,14 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, nin); return -1; } - for (i=nin; i < nargs; i++) { + for (i = nin; i < nargs; i++) { not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); if (not_all_none) { break; } } if (not_all_none) { - if (nargs - nin == nout) - { + if (nargs - nin == nout) { obj = PyTuple_GetSlice(args, nin, nargs); } else { @@ -96,23 +108,72 @@ normalize___call___args(PyUFuncObject *ufunc, PyObject *args, Py_DECREF(obj); } } - return 0; + /* finally, ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); } static int -normalize_reduce_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) +normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) { /* * ufunc.reduce(a[, axis, dtype, out, keepdims]) + */ + int nargs = PyTuple_GET_SIZE(args); + int i; + PyObject *obj; + static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + + if (nargs < 1 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduce() takes from 1 to 5 positional " + "arguments but %d were given", nargs); + return -1; + } + *normal_args = PyTuple_GetSlice(args, 0, 1); + if (*normal_args == NULL) { + return -1; + } + + for (i = 1; i < nargs; i++) { + if (PyDict_GetItemString(*normal_kwds, kwlist[i])) { + PyErr_Format(PyExc_TypeError, + "argument given by name ('%s') and position (%d)", + kwlist[i], i); + return -1; + } + obj = PyTuple_GET_ITEM(args, i); + if (obj != Py_None) { + if (i == 3) { + obj = PyTuple_GetSlice(args, 3, 4); + } + PyDict_SetItemString(*normal_kwds, kwlist[i], obj); + if (i == 3) { + Py_DECREF(obj); + } + } + } + return 0; +} + +static int +normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, + PyObject **normal_args, PyObject **normal_kwds) +{ + /* * ufunc.accumulate(a[, axis, dtype, out]) - * the number of arguments has been checked in PyUFunc_GenericReduction. */ int nargs = PyTuple_GET_SIZE(args); int i; PyObject *obj; static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; + if (nargs < 1 || nargs > 4) { + PyErr_Format(PyExc_TypeError, + "ufunc.accumulate() takes from 1 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } *normal_args = PyTuple_GetSlice(args, 0, 1); if (*normal_args == NULL) { return -1; @@ -152,6 +213,12 @@ normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, PyObject *obj; static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; + if (nargs < 2 || nargs > 5) { + PyErr_Format(PyExc_TypeError, + "ufunc.reduceat() takes from 2 to 4 positional " + "arguments but %d were given", nargs); + return -1; + } /* a and indicies */ *normal_args = PyTuple_GetSlice(args, 0, 2); if (*normal_args == NULL) { @@ -181,14 +248,36 @@ normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, static int normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) + PyObject **normal_args, PyObject **normal_kwds) { /* - * ufunc.outer(A, B) - * This has no kwds so we don't need to do any kwd stuff. + * ufunc.outer(*args, **kwds) + * all positional arguments should be inputs. + * for the keywords, we only need to check 'sig' vs 'signature'. */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - return (*normal_args == NULL); + int nin = ufunc->nin; + int nargs = PyTuple_GET_SIZE(args); + + if (nargs < nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() missing %d of %d required positional " + "argument(s)", nin - nargs, nin); + return -1; + } + if (nargs > nin) { + PyErr_Format(PyExc_TypeError, + "ufunc.outer() takes %d arguments but %d were given", + nin, nargs); + return -1; + } + + *normal_args = PyTuple_GetSlice(args, 0, nin); + if (*normal_args == NULL) { + return -1; + } + + /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ + return normalize_signature_keyword(*normal_kwds); } static int @@ -198,6 +287,12 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, /* ufunc.at(a, indices[, b]) */ int nargs = PyTuple_GET_SIZE(args); + if (nargs < 2 || nargs > 3) { + PyErr_Format(PyExc_TypeError, + "ufunc.at() takes from 2 to 3 positional " + "arguments but %d were given", nargs); + return -1; + } *normal_args = PyTuple_GetSlice(args, 0, nargs); return (*normal_args == NULL); } @@ -334,11 +429,15 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, status = normalize___call___args(ufunc, args, &normal_args, &normal_kwds); } - /* ufunc.reduce and ufunc.accumulate */ - else if ((strcmp(method, "reduce") == 0) || - (strcmp(method, "accumulate") == 0)) { - status = normalize_reduce_accumulate_args(ufunc, args, &normal_args, - &normal_kwds); + /* ufunc.reduce */ + else if (strcmp(method, "reduce") == 0) { + status = normalize_reduce_args(ufunc, args, &normal_args, + &normal_kwds); + } + /* ufunc.accumulate */ + else if (strcmp(method, "accumulate") == 0) { + status = normalize_accumulate_args(ufunc, args, &normal_args, + &normal_kwds); } /* ufunc.reduceat */ else if (strcmp(method, "reduceat") == 0) { diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 1bc7e1109c8f..137a93781a93 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -5067,6 +5067,14 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *new_args, *tmp; PyObject *shape1, *shape2, *newshape; + errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); + if (errval) { + return NULL; + } + else if (override) { + return override; + } + if (ufunc->core_enabled) { PyErr_Format(PyExc_TypeError, "method outer is not allowed in ufunc with non-trivial"\ @@ -5086,15 +5094,6 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return NULL; } - /* Note: `nin` and `nout` are not used in the normalization */ - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - tmp = PySequence_GetItem(args, 0); if (tmp == NULL) { return NULL; @@ -5164,7 +5163,6 @@ ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin` and `nout`, the last two arguments, are not actually used */ errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override); if (errval) { return NULL; @@ -5181,7 +5179,6 @@ ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin` and `nout`, the last two arguments, are not actually used */ errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override); if (errval) { return NULL; @@ -5198,7 +5195,6 @@ ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) int errval; PyObject *override = NULL; - /* `nin` and `nout`, the last two arguments, are not actually used */ errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override); if (errval) { return NULL; @@ -5263,7 +5259,6 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) char * err_msg = NULL; NPY_BEGIN_THREADS_DEF; - /* `nin` and `nout`, the last two arguments, are not actually used */ errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override); if (errval) { return NULL; diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 97e893c681df..ff85e651265e 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1698,6 +1698,11 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_equal(res[3], (1, a)) assert_equal(res[4], {'foo': 'bar', 'answer': 42}) + # __call__, wrong args + assert_raises(TypeError, np.multiply, a) + assert_raises(TypeError, np.multiply, a, a, a, a) + assert_raises(TypeError, np.multiply, a, a, sig='a', signature='a') + # reduce, positional args res = np.multiply.reduce(a, 'axis0', 'dtype0', 'out0', 'keep0') assert_equal(res[0], a) @@ -1808,6 +1813,10 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_equal(res[3], (a, 42)) assert_equal(res[4], {}) + # outer, wrong args + assert_raises(TypeError, np.multiply.outer, a) + assert_raises(TypeError, np.multiply.outer, a, a, a, a) + # at res = np.multiply.at(a, [4, 2], 'b0') assert_equal(res[0], a) @@ -1815,6 +1824,10 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_equal(res[2], 'at') assert_equal(res[3], (a, [4, 2], 'b0')) + # at, wrong args + assert_raises(TypeError, np.multiply.at, a) + assert_raises(TypeError, np.multiply.at, a, a, a, a) + def test_ufunc_override_out(self): class A(object): From 256a8ae75fc36f7d4531557f9572a046508afa07 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sat, 22 Apr 2017 18:01:35 -0700 Subject: [PATCH 41/43] BUG: Fix ArrayLike(NDArrayOperatorsMixin) operations with object() --- numpy/lib/mixins.py | 17 +++++++++++------ numpy/lib/tests/test_mixins.py | 26 ++++++++++++++++++++------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 877a11039b4b..2deb58827729 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -88,18 +88,23 @@ def __init__(self, value): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out = kwargs.get('out', ()) for x in inputs + out: - # Only support operations with instances of _HANDLED_TYPES - # and superclass instances of this type + # Only support operations with instances of _HANDLED_TYPES, + # or instances of ArrayLike that are superclasses of this + # object's type. if not (isinstance(x, self._HANDLED_TYPES) or - isinstance(self, type(x))): + (isinstance(x, ArrayLike) and + isinstance(self, type(x)))): return NotImplemented - # Defer to the implementation of the ufunc on unwrapped values - inputs = tuple(x.value if isinstance(self, type(x)) else x + # Defer to the implementation of the ufunc on unwrapped values. + # Use ArrayLike instead of type(self) for isinstance to allow + # subclasses that don't override __array_ufunc__ to handle + # ArrayLike objects. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x for x in inputs) if out: kwargs['out'] = tuple( - x.value if isinstance(self, type(x)) else x + x.value if isinstance(x, ArrayLike) else x for x in out) result = getattr(ufunc, method)(*inputs, **kwargs) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py index f45a3c6617e8..bca974fc5571 100644 --- a/numpy/lib/tests/test_mixins.py +++ b/numpy/lib/tests/test_mixins.py @@ -26,18 +26,23 @@ def __init__(self, value): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out = kwargs.get('out', ()) for x in inputs + out: - # Only support operations with instances of _HANDLED_TYPES - # and superclass instances of this type + # Only support operations with instances of _HANDLED_TYPES, + # or instances of ArrayLike that are superclasses of this + # object's type. if not (isinstance(x, self._HANDLED_TYPES) or - isinstance(self, type(x))): + (isinstance(x, ArrayLike) and + isinstance(self, type(x)))): return NotImplemented - # Defer to the implementation of the ufunc on unwrapped values - inputs = tuple(x.value if isinstance(self, type(x)) else x + # Defer to the implementation of the ufunc on unwrapped values. + # Use ArrayLike instead of type(self) for isinstance to allow + # subclasses that don't override __array_ufunc__ to handle + # ArrayLike objects. + inputs = tuple(x.value if isinstance(x, ArrayLike) else x for x in inputs) if out: kwargs['out'] = tuple( - x.value if isinstance(self, type(x)) else x + x.value if isinstance(x, ArrayLike) else x for x in out) result = getattr(ufunc, method)(*inputs, **kwargs) @@ -128,6 +133,15 @@ class SubArrayLike(ArrayLike): _assert_equal_type_and_value(x + y, y) _assert_equal_type_and_value(y + x, y) + def test_object(self): + x = ArrayLike(0) + obj = object() + assert_equal(x.__add__(obj), NotImplemented) + with assert_raises(TypeError): + x + obj + with assert_raises(TypeError): + obj + x + def test_unary_methods(self): array = np.array([-1, 0, 1, 2]) array_like = ArrayLike(array) From 3272a860129a7192a0e499c59e273da3dd35d998 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 24 Apr 2017 13:58:49 -0700 Subject: [PATCH 42/43] ENH: Better error message for __array_ufunc__ not implemented * ENH: Better error message for __array_ufunc__ not implemented New behavior: >>> import numpy as np >>> class Dummy: ... def __array_ufunc__(self, *args, **kwargs): ... return NotImplemented >>> np.negative(Dummy()) TypeError: operand type(s) do not implement __array_ufunc__( , '__call__', <__main__.Dummy object at 0x1106df8d0>): 'Dummy' * check for null errmsg_formatter --- numpy/core/_internal.py | 12 ++++++++++++ numpy/core/src/umath/override.c | 18 +++++++++++++++--- numpy/core/tests/test_umath.py | 22 ++++++++++++++++++++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index f25159d8e95a..01741cd1afb6 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -645,3 +645,15 @@ def __init__(self, axis, ndim=None, msg_prefix=None): msg = "{}: {}".format(msg_prefix, msg) super(AxisError, self).__init__(msg) + + +def array_ufunc_errmsg_formatter(ufunc, method, *inputs, **kwargs): + """ Format the error message for when __array_ufunc__ gives up. """ + args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] + + ['{}={!r}'.format(k, v) + for k, v in kwargs.items()]) + args = inputs + kwargs.get('out', ()) + types_string = ', '.join(repr(type(arg).__name__) for arg in args) + return ('operand type(s) do not implement __array_ufunc__' + '({!r}, {!r}, {}): {}' + .format(ufunc, method, args_string, types_string)) diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index d059622c5edb..1faf2568b4f1 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -353,7 +353,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, out = PyDict_GetItemString(normal_kwds, "out"); if (out != NULL) { int nout = ufunc->nout; - + if (PyTuple_Check(out)) { int all_none = 1; @@ -528,8 +528,20 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* Check if there is a method left to call */ if (!override_obj) { /* No acceptable override found. */ - PyErr_SetString(PyExc_TypeError, - "__array_ufunc__ not implemented for this type."); + static PyObject *errmsg_formatter = NULL; + PyObject *errmsg; + + npy_cache_import("numpy.core._internal", + "array_ufunc_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + errmsg = PyObject_Call(errmsg_formatter, override_args, + normal_kwds); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } goto fail; } diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index ff85e651265e..41108ab5f8d2 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -3,6 +3,7 @@ import sys import platform import warnings +import fnmatch import itertools from numpy.testing.utils import _gen_alignment_data @@ -11,8 +12,9 @@ import numpy as np from numpy.testing import ( TestCase, run_module_suite, assert_, assert_equal, assert_raises, - assert_array_equal, assert_almost_equal, assert_array_almost_equal, - dec, assert_allclose, assert_no_warnings, suppress_warnings + assert_raises_regex, assert_array_equal, assert_almost_equal, + assert_array_almost_equal, dec, assert_allclose, assert_no_warnings, + suppress_warnings ) @@ -1896,6 +1898,22 @@ def __array_ufunc__(self, *a, **kwargs): assert_raises(ValueError, np.negative, a) assert_raises(ValueError, np.divide, 1., a) + def test_ufunc_override_not_implemented(self): + + class A(object): + __array_ufunc__ = None + + msg = ("operand type(s) do not implement __array_ufunc__(" + ", '__call__', <*>): 'A'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.negative(A()) + + msg = ("operand type(s) do not implement __array_ufunc__(" + ", '__call__', <*>, , out=(1,)): " + "'A', 'object', 'int'") + with assert_raises_regex(TypeError, fnmatch.translate(msg)): + np.add(A(), object(), out=1) + def test_gufunc_override(self): # gufunc are just ufunc instances, but follow a different path, # so check __array_ufunc__ overrides them properly. From 32221dfb553980e34a398c71891c7dcdfaf2f477 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 27 Apr 2017 12:17:06 -0700 Subject: [PATCH 43/43] ENH: NDArrayOperatorsMixin calls ufuncs directly, like ndarray * ENH: NDArrayOperatorsMixin calls ufuncs directly, like ndarray Per our discussion in https://github.com/numpy/numpy/pull/8247#discussion_r112825050 * add back in accidentally dropped __repr__ --- numpy/lib/mixins.py | 59 +++++++++++++++++----------------- numpy/lib/tests/test_mixins.py | 17 ++++------ 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 2deb58827729..21e4b346f37d 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -5,53 +5,54 @@ from numpy.core import umath as um -# None of this module should be exposed in top-level NumPy module. +# Nothing should be exposed in the top-level NumPy module. __all__ = [] +def _disables_array_ufunc(obj): + """True when __array_ufunc__ is set to None.""" + try: + return obj.__array_ufunc__ is None + except AttributeError: + return False + + def _binary_method(ufunc): + """Implement a forward binary method with a ufunc, e.g., __add__.""" def func(self, other): - try: - if other.__array_ufunc__ is None: - return NotImplemented - except AttributeError: - pass - return self.__array_ufunc__(ufunc, '__call__', self, other) + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(self, other) return func def _reflected_binary_method(ufunc): + """Implement a reflected binary method with a ufunc, e.g., __radd__.""" def func(self, other): - try: - if other.__array_ufunc__ is None: - return NotImplemented - except AttributeError: - pass - return self.__array_ufunc__(ufunc, '__call__', other, self) + if _disables_array_ufunc(other): + return NotImplemented + return ufunc(other, self) return func def _inplace_binary_method(ufunc): + """Implement an in-place binary method with a ufunc, e.g., __iadd__.""" def func(self, other): - result = self.__array_ufunc__( - ufunc, '__call__', self, other, out=(self,)) - if result is NotImplemented: - raise TypeError('unsupported operand types for in-place ' - 'arithmetic: %s and %s' - % (type(self).__name__, type(other).__name__)) - return result + return ufunc(self, other, out=(self,)) return func def _numeric_methods(ufunc): + """Implement forward, reflected and inplace binary methods with a ufunc.""" return (_binary_method(ufunc), _reflected_binary_method(ufunc), _inplace_binary_method(ufunc)) def _unary_method(ufunc): + """Implement a unary special method with a ufunc.""" def func(self): - return self.__array_ufunc__(ufunc, '__call__', self) + return ufunc(self) return func @@ -88,18 +89,14 @@ def __init__(self, value): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out = kwargs.get('out', ()) for x in inputs + out: - # Only support operations with instances of _HANDLED_TYPES, - # or instances of ArrayLike that are superclasses of this - # object's type. - if not (isinstance(x, self._HANDLED_TYPES) or - (isinstance(x, ArrayLike) and - isinstance(self, type(x)))): + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): return NotImplemented # Defer to the implementation of the ufunc on unwrapped values. - # Use ArrayLike instead of type(self) for isinstance to allow - # subclasses that don't override __array_ufunc__ to handle - # ArrayLike objects. inputs = tuple(x.value if isinstance(x, ArrayLike) else x for x in inputs) if out: @@ -138,6 +135,8 @@ def __repr__(self): with arbitrary, unrecognized types. This ensures that interactions with ArrayLike preserve a well-defined casting hierarchy. """ + # Like np.ndarray, this mixin class implements "Option 1" from the ufunc + # overrides NEP. # comparisons don't have reflected and in-place versions __lt__ = _binary_method(um.less) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py index bca974fc5571..57c4a4cd80e0 100644 --- a/numpy/lib/tests/test_mixins.py +++ b/numpy/lib/tests/test_mixins.py @@ -26,18 +26,14 @@ def __init__(self, value): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out = kwargs.get('out', ()) for x in inputs + out: - # Only support operations with instances of _HANDLED_TYPES, - # or instances of ArrayLike that are superclasses of this - # object's type. - if not (isinstance(x, self._HANDLED_TYPES) or - (isinstance(x, ArrayLike) and - isinstance(self, type(x)))): + # Only support operations with instances of _HANDLED_TYPES. + # Use ArrayLike instead of type(self) for isinstance to + # allow subclasses that don't override __array_ufunc__ to + # handle ArrayLike objects. + if not isinstance(x, self._HANDLED_TYPES + (ArrayLike,)): return NotImplemented # Defer to the implementation of the ufunc on unwrapped values. - # Use ArrayLike instead of type(self) for isinstance to allow - # subclasses that don't override __array_ufunc__ to handle - # ArrayLike objects. inputs = tuple(x.value if isinstance(x, ArrayLike) else x for x in inputs) if out: @@ -136,11 +132,12 @@ class SubArrayLike(ArrayLike): def test_object(self): x = ArrayLike(0) obj = object() - assert_equal(x.__add__(obj), NotImplemented) with assert_raises(TypeError): x + obj with assert_raises(TypeError): obj + x + with assert_raises(TypeError): + x += obj def test_unary_methods(self): array = np.array([-1, 0, 1, 2])