From 47086158cb00a151b67c442ae759ce230ec0de34 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 20 Sep 2012 22:02:18 +0100 Subject: [PATCH 1/4] ENH: More capable test functions for warnings 1) New function assert_no_warnings 2) Make assert_warns and assert_no_warnings pass through the function's return value on success, so that it can be checked as well. --- numpy/testing/tests/test_utils.py | 6 ++++- numpy/testing/utils.py | 40 ++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index bff8b50ab5c0..23b2f8e7b228 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -317,11 +317,15 @@ class TestWarns(unittest.TestCase): def test_warn(self): def f(): warnings.warn("yo") + return 3 before_filters = sys.modules['warnings'].filters[:] - assert_warns(UserWarning, f) + assert_equal(assert_warns(UserWarning, f), 3) after_filters = sys.modules['warnings'].filters + assert_raises(AssertionError, assert_no_warnings, f) + assert_equal(assert_no_warnings(lambda x: x, 1), 1) + # Check that the warnings state is unchanged assert_equal(before_filters, after_filters, "assert_warns does not preserver warnings state") diff --git a/numpy/testing/utils.py b/numpy/testing/utils.py index ffce2eefc836..16ed0f803a66 100644 --- a/numpy/testing/utils.py +++ b/numpy/testing/utils.py @@ -16,7 +16,8 @@ 'decorate_methods', 'jiffies', 'memusage', 'print_assert_equal', 'raises', 'rand', 'rundocs', 'runstring', 'verbose', 'measure', 'assert_', 'assert_array_almost_equal_nulp', - 'assert_array_max_ulp', 'assert_warns', 'assert_allclose'] + 'assert_array_max_ulp', 'assert_warns', 'assert_no_warnings', + 'assert_allclose'] verbose = 0 @@ -1464,7 +1465,7 @@ def assert_warns(warning_class, func, *args, **kw): Returns ------- - None + The value returned by `func`. """ @@ -1474,7 +1475,7 @@ def assert_warns(warning_class, func, *args, **kw): l = ctx.__enter__() warnings.simplefilter('always') try: - func(*args, **kw) + result = func(*args, **kw) if not len(l) > 0: raise AssertionError("No warning raised when calling %s" % func.__name__) @@ -1483,3 +1484,36 @@ def assert_warns(warning_class, func, *args, **kw): "%s( is %s)" % (func.__name__, warning_class, l[0])) finally: ctx.__exit__() + return result + +def assert_no_warnings(func, *args, **kw): + """ + Fail if the given callable produces any warnings. + + Parameters + ---------- + func : callable + The callable to test. + \\*args : Arguments + Arguments passed to `func`. + \\*\\*kwargs : Kwargs + Keyword arguments passed to `func`. + + Returns + ------- + The value returned by `func`. + + """ + # XXX: once we may depend on python >= 2.6, this can be replaced by the + # warnings module context manager. + ctx = WarningManager(record=True) + l = ctx.__enter__() + warnings.simplefilter('always') + try: + result = func(*args, **kw) + if len(l) > 0: + raise AssertionError("Got warnings when calling %s: %s" + % (func.__name__, l)) + finally: + ctx.__exit__() + return result From cea0a209875be753a74b8c7bb02aa9531726ee98 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 20 Sep 2012 22:03:31 +0100 Subject: [PATCH 2/4] FIX: Transition scheme for safer in-place ufunc operations In numpy 1.6 and earlier, if you do np.add(int_arr, float_arr, out=int_arr) or int_arr += float_arr then the result will be silently truncated to integer values. This often produces bugs, because it's easy to accidentally end up with an integer array and not realize it. Therefore, there seems to be consensus that we should switch to using same_kind casting by default for in-place ufunc operations. However, just switching this (as was done initially during the 1.7 development cycle) breaks a lot of code, which is rude and violates our deprecation policy. This commit instead adds a special temporary casting rule which acts like "unsafe", but also checks whether each operation would be allowed under "same_kind" rules and issues a DeprecationWarning if not. It also moves NPY_DEFAULT_ASSIGN_CASTING into the formal API instead of leaving it as a #define. This way we can change it later, and any code which references it and is compiled against this version of numpy will automatically switch to whatever we change it too. This avoids the situation where we want to remove the temporary magic value we're using to create DeprecationWarnings now, but can't because it would be an ABI break. --- doc/release/1.7.0-notes.rst | 12 +++++--- doc/source/reference/ufuncs.rst | 6 ++++ numpy/core/code_generators/numpy_api.py | 2 ++ numpy/core/include/numpy/ndarraytypes.h | 9 +++--- numpy/core/src/multiarray/common.c | 14 ++++++++++ numpy/core/src/multiarray/convert_datatype.c | 29 ++++++++++++++++++++ numpy/core/tests/test_ufunc.py | 18 ++++++++++++ 7 files changed, 82 insertions(+), 8 deletions(-) diff --git a/doc/release/1.7.0-notes.rst b/doc/release/1.7.0-notes.rst index f8f54219cac3..c38f6eff1ec8 100644 --- a/doc/release/1.7.0-notes.rst +++ b/doc/release/1.7.0-notes.rst @@ -33,10 +33,14 @@ np.diagonal, numpy 1.7 produces a FutureWarning if it detects that you may be attemping to write to such an array. See the documentation for array indexing for details. -The default casting rule for UFunc out= parameters has been changed from -'unsafe' to 'same_kind'. Most usages which violate the 'same_kind' -rule are likely bugs, so this change may expose previously undetected -errors in projects that depend on NumPy. +In a future version of numpy, the default casting rule for UFunc out= +parameters will be changed from 'unsafe' to 'same_kind'. (This also +applies to in-place operations like a += b, which is equivalent to +np.add(a, b, out=a).) Most usages which violate the 'same_kind' rule +are likely bugs, so this change may expose previously undetected +errors in projects that depend on NumPy. In this version of numpy, +such usages will continue to succeed, but will raise a +DeprecationWarning. Full-array boolean indexing has been optimized to use a different, optimized code path. This code path should produce the same results, diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 295d52ef4673..afcb1302b32e 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -309,6 +309,12 @@ advanced usage and will not typically be used. 'equiv', 'safe', 'same_kind', or 'unsafe'. See :func:`can_cast` for explanations of the parameter values. + In a future version of numpy, this argument will default to + 'same_kind'. As part of this transition, starting in version 1.7, + ufuncs will produce a DeprecationWarning for calls which are + allowed under the 'unsafe' rules, but not under the 'same_kind' + rules. + *order* .. versionadded:: 1.6 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index c49c3c346fbf..cb598880be1b 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -14,10 +14,12 @@ multiarray_global_vars = { 'NPY_NUMUSERTYPES': 7, + 'NPY_DEFAULT_ASSIGN_CASTING': 292, } multiarray_global_vars_types = { 'NPY_NUMUSERTYPES': 'int', + 'NPY_DEFAULT_ASSIGN_CASTING': 'NPY_CASTING', } multiarray_scalar_bool_values = { diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 954303352f82..523601570b65 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -199,11 +199,12 @@ typedef enum { /* Allow safe casts or casts within the same kind */ NPY_SAME_KIND_CASTING=3, /* Allow any casts */ - NPY_UNSAFE_CASTING=4 -} NPY_CASTING; + NPY_UNSAFE_CASTING=4, -/* The default casting to use for typical assignment operations */ -#define NPY_DEFAULT_ASSIGN_CASTING NPY_SAME_KIND_CASTING + /* Temporary internal definition only, will be removed in upcoming + release, see below */ + NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND = 100, +} NPY_CASTING; typedef enum { NPY_CLIP=0, diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 5ab8f92bc76b..917946b484f1 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -13,6 +13,20 @@ #include "common.h" #include "buffer.h" +/* The casting to use for implicit assignment operations resulting from + * in-place operations (like +=) and out= arguments. (Notice that this + * variable is misnamed, but it's part of the public API so I'm not sure we + * can just change it. Maybe someone should try and see if anyone notices. + */ +/* In numpy 1.6 and earlier, this was NPY_UNSAFE_CASTING. In a future + * release, it will become NPY_SAME_KIND_CASTING. Right now, during the + * transitional period, we continue to follow the NPY_UNSAFE_CASTING rules (to + * avoid breaking people's code), but we also check for whether the cast would + * be allowed under the NPY_SAME_KIND_CASTING rules, and if not we issue a + * warning (that people's code will be broken in a future release.) + */ +NPY_NO_EXPORT NPY_CASTING NPY_DEFAULT_ASSIGN_CASTING = NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND; + NPY_NO_EXPORT PyArray_Descr * _array_find_python_scalar_type(PyObject *op) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index de7468c51e00..b4f20c000b0f 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -503,12 +503,41 @@ type_num_unsigned_to_signed(int type_num) } } +/* NOTE: once the UNSAFE_CASTING -> SAME_KIND_CASTING transition is over, + * we should remove NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND + * and PyArray_CanCastTypeTo_impl should be renamed back to + * PyArray_CanCastTypeTo. + */ +static npy_bool +PyArray_CanCastTypeTo_impl(PyArray_Descr *from, PyArray_Descr *to, + NPY_CASTING casting); + /*NUMPY_API * Returns true if data of type 'from' may be cast to data of type * 'to' according to the rule 'casting'. */ NPY_NO_EXPORT npy_bool PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, + NPY_CASTING casting) +{ + if (casting == NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND) { + npy_bool unsafe_ok, same_kind_ok; + unsafe_ok = PyArray_CanCastTypeTo_impl(from, to, NPY_UNSAFE_CASTING); + same_kind_ok = PyArray_CanCastTypeTo_impl(from, to, + NPY_SAME_KIND_CASTING); + if (unsafe_ok && !same_kind_ok) { + DEPRECATE("Implicitly casting between incompatible kinds. In " + "a future numpy release, this will become an error. " + "Use casting=\"unsafe\" if this is intentional."); + } + return unsafe_ok; + } else { + return PyArray_CanCastTypeTo_impl(from, to, casting); + } +} + +static npy_bool +PyArray_CanCastTypeTo_impl(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting) { /* If unsafe casts are allowed */ diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 53928129f84f..fb6b586be607 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -742,5 +742,23 @@ def t(expect, func, n, m): uf.accumulate(np.zeros((30, 30)), axis=0) uf.accumulate(np.zeros((0, 0)), axis=0) + def test_safe_casting(self): + # In old numpy's, any casting was allowed for in-place operations. In + # future numpy's, only same_kind casting will be allowed by + # default. + a = np.array([1, 2, 3], dtype=int) + # Non-in-place addition is fine + assert_array_equal(assert_no_warnings(np.add, a, 1.1), + [2.1, 3.1, 4.1]) + assert_warns(DeprecationWarning, np.add, a, 1.1, out=a) + assert_array_equal(a, [2, 3, 4]) + def add_inplace(a, b): + a += b + assert_warns(DeprecationWarning, add_inplace, a, 1.1) + assert_array_equal(a, [3, 4, 5]) + # Make sure that explicitly overriding the warning is allowed: + assert_no_warnings(np.add, a, 1.1, out=a, casting="unsafe") + assert_array_equal(a, [4, 5, 6]) + if __name__ == "__main__": run_module_suite() From c038fe59a8c355fee50a6c2ac236e750e0ad0ad5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 20 Sep 2012 23:30:29 +0100 Subject: [PATCH 3/4] STY: fix up style and a few wordings on previous commit --- numpy/core/include/numpy/ndarraytypes.h | 6 ++++-- numpy/core/src/multiarray/common.c | 6 ++++-- numpy/core/src/multiarray/convert_datatype.c | 8 +++++--- numpy/core/tests/test_ufunc.py | 6 +++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 523601570b65..93d561c7c511 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -201,8 +201,10 @@ typedef enum { /* Allow any casts */ NPY_UNSAFE_CASTING=4, - /* Temporary internal definition only, will be removed in upcoming - release, see below */ + /* + * Temporary internal definition only, will be removed in upcoming + * release, see below + * */ NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND = 100, } NPY_CASTING; diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 917946b484f1..7b8177c5cb12 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -13,12 +13,14 @@ #include "common.h" #include "buffer.h" -/* The casting to use for implicit assignment operations resulting from +/* + * The casting to use for implicit assignment operations resulting from * in-place operations (like +=) and out= arguments. (Notice that this * variable is misnamed, but it's part of the public API so I'm not sure we * can just change it. Maybe someone should try and see if anyone notices. */ -/* In numpy 1.6 and earlier, this was NPY_UNSAFE_CASTING. In a future +/* + * In numpy 1.6 and earlier, this was NPY_UNSAFE_CASTING. In a future * release, it will become NPY_SAME_KIND_CASTING. Right now, during the * transitional period, we continue to follow the NPY_UNSAFE_CASTING rules (to * avoid breaking people's code), but we also check for whether the cast would diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index b4f20c000b0f..586b85b1e9be 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -503,7 +503,8 @@ type_num_unsigned_to_signed(int type_num) } } -/* NOTE: once the UNSAFE_CASTING -> SAME_KIND_CASTING transition is over, +/* + * NOTE: once the UNSAFE_CASTING -> SAME_KIND_CASTING transition is over, * we should remove NPY_INTERNAL_UNSAFE_CASTING_BUT_WARN_UNLESS_SAME_KIND * and PyArray_CanCastTypeTo_impl should be renamed back to * PyArray_CanCastTypeTo. @@ -527,11 +528,12 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, NPY_SAME_KIND_CASTING); if (unsafe_ok && !same_kind_ok) { DEPRECATE("Implicitly casting between incompatible kinds. In " - "a future numpy release, this will become an error. " + "a future numpy release, this will raise an error. " "Use casting=\"unsafe\" if this is intentional."); } return unsafe_ok; - } else { + } + else { return PyArray_CanCastTypeTo_impl(from, to, casting); } } diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index fb6b586be607..84d304595e3d 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -743,9 +743,9 @@ def t(expect, func, n, m): uf.accumulate(np.zeros((0, 0)), axis=0) def test_safe_casting(self): - # In old numpy's, any casting was allowed for in-place operations. In - # future numpy's, only same_kind casting will be allowed by - # default. + # In old numpy's, unsafe casting was allowed for in-place + # operations. In future numpy's, only same_kind casting will be + # allowed by default. a = np.array([1, 2, 3], dtype=int) # Non-in-place addition is fine assert_array_equal(assert_no_warnings(np.add, a, 1.1), From f18987a69c297b5602b00c22b9759d2ece4a7bf1 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 21 Sep 2012 00:25:11 +0100 Subject: [PATCH 4/4] STY: another wording tweak (thanks to @charris) --- numpy/core/tests/test_ufunc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 84d304595e3d..57fd66892938 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -743,9 +743,9 @@ def t(expect, func, n, m): uf.accumulate(np.zeros((0, 0)), axis=0) def test_safe_casting(self): - # In old numpy's, unsafe casting was allowed for in-place - # operations. In future numpy's, only same_kind casting will be - # allowed by default. + # In old versions of numpy, in-place operations used the 'unsafe' + # casting rules. In some future version, 'same_kind' will become the + # default. a = np.array([1, 2, 3], dtype=int) # Non-in-place addition is fine assert_array_equal(assert_no_warnings(np.add, a, 1.1),