From bfe0921f9bdd6b347982a9c998b8b1890040ba5b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sat, 27 Oct 2018 12:23:15 -0700 Subject: [PATCH 1/4] DEP: deprecate passing a generator to stack functions Fixes gh-12263 We can't support generators with dispatch for ``__array_function__``. --- numpy/core/shape_base.py | 27 +++++++++++++++++++-------- numpy/core/tests/test_shape_base.py | 16 ++++++++++++++-- numpy/lib/shape_base.py | 5 +++-- numpy/lib/tests/test_shape_base.py | 9 +++++++++ 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 71a23f438c74..4cb7a30f4ddc 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -5,6 +5,8 @@ import functools import operator +import types +import warnings from . import numeric as _nx from . import overrides @@ -204,11 +206,23 @@ def atleast_3d(*arys): return res -def _vstack_dispatcher(tup): - return tup +def _arrays_for_stack_dispatcher(arrays, stacklevel=4): + if isinstance(arrays, types.GeneratorType): + warnings.warn('arrays to stack should be passed as a sequence, not a ' + 'generator. Support for generators is deprecated as of ' + 'NumPy 1.16 and will raise an error in the future. ' + 'Note also that dispatch with __array_function__ is not ' + 'supported when passing arrays as a generator.', + FutureWarning, stacklevel=stacklevel) + return () + return arrays -@array_function_dispatch(_vstack_dispatcher) +def _vhstack_dispatcher(tup): + return _arrays_for_stack_dispatcher(tup) + + +@array_function_dispatch(_vhstack_dispatcher) def vstack(tup): """ Stack arrays in sequence vertically (row wise). @@ -264,11 +278,7 @@ def vstack(tup): return _nx.concatenate([atleast_2d(_m) for _m in tup], 0) -def _hstack_dispatcher(tup): - return tup - - -@array_function_dispatch(_hstack_dispatcher) +@array_function_dispatch(_vhstack_dispatcher) def hstack(tup): """ Stack arrays in sequence horizontally (column wise). @@ -325,6 +335,7 @@ def hstack(tup): def _stack_dispatcher(arrays, axis=None, out=None): + arrays = _arrays_for_stack_dispatcher(arrays, stacklevel=6) for a in arrays: yield a if out is not None: diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 9bedd86708ba..70d217c9115c 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -10,7 +10,7 @@ _block_concatenate, _block_slicing) from numpy.testing import ( assert_, assert_raises, assert_array_equal, assert_equal, - assert_raises_regex, assert_almost_equal + assert_raises_regex, assert_warns, assert_almost_equal ) from numpy.compat import long @@ -155,6 +155,10 @@ def test_2D_array(self): desired = array([[1, 1], [2, 2]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + hstack((np.arange(3) for _ in range(2))) + class TestVstack(object): def test_non_iterable(self): @@ -191,6 +195,10 @@ def test_2D_array2(self): desired = array([[1, 2], [1, 2]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + vstack((np.arange(3) for _ in range(2))) + class TestConcatenate(object): def test_returns_copy(self): @@ -354,7 +362,7 @@ def test_stack(): arrays = [np.random.randn(3, 4) for _ in range(10)] axes = [0, 1, 2, -1, -2, -3] expected_shapes = [(10, 3, 4), (3, 10, 4), (3, 4, 10), - (3, 4, 10), (3, 10, 4), (10, 3, 4)] + (3, 4, 10), (3, 10, 4), (10, 3, 4)] for axis, expected_shape in zip(axes, expected_shapes): assert_equal(np.stack(arrays, axis).shape, expected_shape) # empty arrays @@ -372,6 +380,10 @@ def test_stack(): stack, [np.zeros((3, 3)), np.zeros(3)], axis=1) assert_raises_regex(ValueError, 'must have the same shape', stack, [np.arange(2), np.arange(3)]) + # generator is deprecated + with assert_warns(FutureWarning): + result = stack((x for x in range(3))) + assert_array_equal(result, np.array([0, 1, 2])) # See for more information on how to parametrize a whole class diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index 00424d55d324..6e7cab3fa923 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -11,6 +11,7 @@ from numpy.core.multiarray import normalize_axis_index from numpy.core import overrides from numpy.core import vstack, atleast_3d +from numpy.core.shape_base import _arrays_for_stack_dispatcher from numpy.lib.index_tricks import ndindex from numpy.matrixlib.defmatrix import matrix # this raises all the right alarm bells @@ -591,7 +592,7 @@ def expand_dims(a, axis): def _column_stack_dispatcher(tup): - return tup + return _arrays_for_stack_dispatcher(tup) @array_function_dispatch(_column_stack_dispatcher) @@ -638,7 +639,7 @@ def column_stack(tup): def _dstack_dispatcher(tup): - return tup + return _arrays_for_stack_dispatcher(tup) @array_function_dispatch(_dstack_dispatcher) diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index a7f5ca7db3ca..e338467f9fb6 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -457,6 +457,7 @@ def test_unequal_split(self): a = np.arange(10) assert_raises(ValueError, split, a, 3) + class TestColumnStack(object): def test_non_iterable(self): assert_raises(TypeError, column_stack, 1) @@ -481,6 +482,10 @@ def test_2D_arrays(self): actual = np.column_stack((a, b)) assert_equal(actual, expected) + def test_generator(self): + with assert_warns(FutureWarning): + column_stack((np.arange(3) for _ in range(2))) + class TestDstack(object): def test_non_iterable(self): @@ -514,6 +519,10 @@ def test_2D_array2(self): desired = np.array([[[1, 1], [2, 2]]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + dstack((np.arange(3) for _ in range(2))) + # array_split has more comprehensive test of splitting. # only do simple test on hsplit, vsplit, and dsplit From ced6475d519552455cc50831052213c1249a909c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sat, 27 Oct 2018 13:09:23 -0700 Subject: [PATCH 2/4] MAINT: warn when passing map to stack functions, too --- numpy/core/shape_base.py | 8 ++++---- numpy/core/tests/test_shape_base.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 4cb7a30f4ddc..22ce47162bc0 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -207,12 +207,12 @@ def atleast_3d(*arys): def _arrays_for_stack_dispatcher(arrays, stacklevel=4): - if isinstance(arrays, types.GeneratorType): - warnings.warn('arrays to stack should be passed as a sequence, not a ' - 'generator. Support for generators is deprecated as of ' + if hasattr(arrays, '__iter__') and not hasattr(arrays, '__getitem__'): + warnings.warn('arrays to stack must be passed as a sequence. Support ' + 'for non-sequence iterables is deprecated as of ' 'NumPy 1.16 and will raise an error in the future. ' 'Note also that dispatch with __array_function__ is not ' - 'supported when passing arrays as a generator.', + 'supported when arrays are not provided as a sequence.', FutureWarning, stacklevel=stacklevel) return () return arrays diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 70d217c9115c..0aebe54d0c0d 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -158,6 +158,8 @@ def test_2D_array(self): def test_generator(self): with assert_warns(FutureWarning): hstack((np.arange(3) for _ in range(2))) + with assert_warns(FutureWarning): + hstack(map(lambda x: x, np.ones((3, 2)))) class TestVstack(object): From c4f853968c862849f159bad5c187310c1aa93e56 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sat, 27 Oct 2018 13:29:26 -0700 Subject: [PATCH 3/4] MAINT: fix test on Python 2 --- numpy/core/tests/test_shape_base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 0aebe54d0c0d..b2c610da60e6 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -1,6 +1,7 @@ from __future__ import division, absolute_import, print_function import warnings +import sys import numpy as np from numpy.core import ( array, arange, atleast_1d, atleast_2d, atleast_3d, block, vstack, hstack, @@ -158,8 +159,10 @@ def test_2D_array(self): def test_generator(self): with assert_warns(FutureWarning): hstack((np.arange(3) for _ in range(2))) - with assert_warns(FutureWarning): - hstack(map(lambda x: x, np.ones((3, 2)))) + if sys.version_info.major > 2: + # map returns a list on Python 2 + with assert_warns(FutureWarning): + hstack(map(lambda x: x, np.ones((3, 2)))) class TestVstack(object): From 00640218fcbc5d71f22550a9368da92358c8de96 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 28 Oct 2018 14:55:54 -0700 Subject: [PATCH 4/4] MAINT: adjust stack deprecation warning per review --- numpy/core/shape_base.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 22ce47162bc0..3edf0824ed09 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -207,12 +207,11 @@ def atleast_3d(*arys): def _arrays_for_stack_dispatcher(arrays, stacklevel=4): - if hasattr(arrays, '__iter__') and not hasattr(arrays, '__getitem__'): - warnings.warn('arrays to stack must be passed as a sequence. Support ' - 'for non-sequence iterables is deprecated as of ' - 'NumPy 1.16 and will raise an error in the future. ' - 'Note also that dispatch with __array_function__ is not ' - 'supported when arrays are not provided as a sequence.', + if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'): + warnings.warn('arrays to stack must be passed as a "sequence" type ' + 'such as list or tuple. Support for non-sequence ' + 'iterables such as generators is deprecated as of ' + 'NumPy 1.16 and will raise an error in the future.', FutureWarning, stacklevel=stacklevel) return () return arrays