From a5c6f0dbfff36956d879010177cfea3eb4f5b04f Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 7 Sep 2017 16:58:19 +0100 Subject: [PATCH 01/23] Simplify block implementation Remove _Recurser class. --- numpy/core/shape_base.py | 200 ++++++++++----------------------------- 1 file changed, 50 insertions(+), 150 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 026ad603a500..d6fa0cbcfd2f 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -365,78 +365,54 @@ def stack(arrays, axis=0, out=None): return _nx.concatenate(expanded_arrays, axis=axis, out=out) -class _Recurser(object): - """ - Utility class for recursing over nested iterables - """ - def __init__(self, recurse_if): - self.recurse_if = recurse_if - - def map_reduce(self, x, f_map=lambda x, **kwargs: x, - f_reduce=lambda x, **kwargs: x, - f_kwargs=lambda **kwargs: kwargs, - **kwargs): - """ - Iterate over the nested list, applying: - * ``f_map`` (T -> U) to items - * ``f_reduce`` (Iterable[U] -> U) to mapped items - - For instance, ``map_reduce([[1, 2], 3, 4])`` is:: - - f_reduce([ - f_reduce([ - f_map(1), - f_map(2) - ]), - f_map(3), - f_map(4) - ]]) - - - State can be passed down through the calls with `f_kwargs`, - to iterables of mapped items. When kwargs are passed, as in - ``map_reduce([[1, 2], 3, 4], **kw)``, this becomes:: - - kw1 = f_kwargs(**kw) - kw2 = f_kwargs(**kw1) - f_reduce([ - f_reduce([ - f_map(1), **kw2) - f_map(2, **kw2) - ], **kw1), - f_map(3, **kw1), - f_map(4, **kw1) - ]], **kw) - """ - def f(x, **kwargs): - if not self.recurse_if(x): - return f_map(x, **kwargs) - else: - next_kwargs = f_kwargs(**kwargs) - return f_reduce(( - f(xi, **next_kwargs) - for xi in x - ), **kwargs) - return f(x, **kwargs) - - def walk(self, x, index=()): - """ - Iterate over x, yielding (index, value, entering), where - - * ``index``: a tuple of indices up to this point - * ``value``: equal to ``x[index[0]][...][index[-1]]``. On the first iteration, is - ``x`` itself - * ``entering``: bool. The result of ``recurse_if(value)`` - """ - do_recurse = self.recurse_if(x) - yield index, x, do_recurse - - if not do_recurse: - return - for i, xi in enumerate(x): - # yield from ... - for v in self.walk(xi, index + (i,)): - yield v +def _check_block_depths_match(arrays, index=[]): + def format_index(index): + idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) + return 'arrays' + idx_str + if isinstance(arrays, tuple): + raise TypeError( + '{} is a tuple. ' + 'Only lists can be used to arrange blocks, and np.block does ' + 'not allow implicit conversion from tuple to ndarray.'.format( + format_index(index) + ) + ) + elif isinstance(arrays, list) and len(arrays) > 0: + indexes = [_check_block_depths_match(arr, index + [i]) + for i, arr in enumerate(arrays)] + + first_index = indexes[0] + for i, index in enumerate(indexes): + if len(index) != len(first_index): + raise ValueError( + "List depths are mismatched. First element was at depth " + "{}, but there is an element at depth {} ({})".format( + len(first_index), + len(index), + format_index(index) + ) + ) + return first_index + elif isinstance(arrays, list) and len(arrays) == 0: + return index + [None] + else: + # We've 'bottomed out' + return index + + +def _block(arrays, depth=0): + if isinstance(arrays, list): + if len(arrays) == 0: + raise ValueError('Lists cannot be empty') + arrs, list_ndims = zip(*(_block(arr, depth+1) for arr in arrays)) + list_ndim = list_ndims[0] + arr_ndim = max(arr.ndim for arr in arrs) + ndim = max(list_ndim, arr_ndim) + arrs = tuple(map(lambda a: _nx.array(a, ndmin=ndim), arrs)) + return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim + else: + # We've 'bottomed out' + return _nx.array(arrays, ndmin=depth), depth def block(arrays): @@ -587,81 +563,5 @@ def block(arrays): """ - def atleast_nd(x, ndim): - x = asanyarray(x) - diff = max(ndim - x.ndim, 0) - return x[(None,)*diff + (Ellipsis,)] - - def format_index(index): - return 'arrays' + ''.join('[{}]'.format(i) for i in index) - - rec = _Recurser(recurse_if=lambda x: type(x) is list) - - # ensure that the lists are all matched in depth - list_ndim = None - any_empty = False - for index, value, entering in rec.walk(arrays): - if type(value) is tuple: - # not strictly necessary, but saves us from: - # - more than one way to do things - no point treating tuples like - # lists - # - horribly confusing behaviour that results when tuples are - # treated like ndarray - raise TypeError( - '{} is a tuple. ' - 'Only lists can be used to arrange blocks, and np.block does ' - 'not allow implicit conversion from tuple to ndarray.'.format( - format_index(index) - ) - ) - if not entering: - curr_depth = len(index) - elif len(value) == 0: - curr_depth = len(index) + 1 - any_empty = True - else: - continue - - if list_ndim is not None and list_ndim != curr_depth: - raise ValueError( - "List depths are mismatched. First element was at depth {}, " - "but there is an element at depth {} ({})".format( - list_ndim, - curr_depth, - format_index(index) - ) - ) - list_ndim = curr_depth - - # do this here so we catch depth mismatches first - if any_empty: - raise ValueError('Lists cannot be empty') - - # convert all the arrays to ndarrays - arrays = rec.map_reduce(arrays, - f_map=asanyarray, - f_reduce=list - ) - - # determine the maximum dimension of the elements - elem_ndim = rec.map_reduce(arrays, - f_map=lambda xi: xi.ndim, - f_reduce=max - ) - ndim = max(list_ndim, elem_ndim) - - # first axis to concatenate along - first_axis = ndim - list_ndim - - # Make all the elements the same dimension - arrays = rec.map_reduce(arrays, - f_map=lambda xi: atleast_nd(xi, ndim), - f_reduce=list - ) - - # concatenate innermost lists on the right, outermost on the left - return rec.map_reduce(arrays, - f_reduce=lambda xs, axis: _nx.concatenate(list(xs), axis=axis), - f_kwargs=lambda axis: dict(axis=axis+1), - axis=first_axis - ) + _check_block_depths_match(arrays) + return _block(arrays)[0] From 2dcc9aa89f3952beec6ffabb3c25a572378c526f Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 7 Sep 2017 20:52:58 +0100 Subject: [PATCH 02/23] np.block style improvements --- numpy/core/shape_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index d6fa0cbcfd2f..a48c15b04bef 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -365,7 +365,7 @@ def stack(arrays, axis=0, out=None): return _nx.concatenate(expanded_arrays, axis=axis, out=out) -def _check_block_depths_match(arrays, index=[]): +def _block_check_depths_match(arrays, index=[]): def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) return 'arrays' + idx_str @@ -378,7 +378,7 @@ def format_index(index): ) ) elif isinstance(arrays, list) and len(arrays) > 0: - indexes = [_check_block_depths_match(arr, index + [i]) + indexes = [_block_check_depths_match(arr, index + [i]) for i, arr in enumerate(arrays)] first_index = indexes[0] @@ -408,7 +408,7 @@ def _block(arrays, depth=0): list_ndim = list_ndims[0] arr_ndim = max(arr.ndim for arr in arrs) ndim = max(list_ndim, arr_ndim) - arrs = tuple(map(lambda a: _nx.array(a, ndmin=ndim), arrs)) + arrs = [_nx.array(a, ndmin=ndim) for a in arrs] return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim else: # We've 'bottomed out' @@ -563,5 +563,5 @@ def block(arrays): """ - _check_block_depths_match(arrays) + _block_check_depths_match(arrays) return _block(arrays)[0] From e787a9fe538156308430e0fc4692fa3bc8ff5f92 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Fri, 8 Sep 2017 12:41:13 +0100 Subject: [PATCH 03/23] Reflect asanyarray behaviour in block array kwargs set to copy=False, subok=True --- numpy/core/shape_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index a48c15b04bef..451ed4319b86 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -408,11 +408,11 @@ def _block(arrays, depth=0): list_ndim = list_ndims[0] arr_ndim = max(arr.ndim for arr in arrs) ndim = max(list_ndim, arr_ndim) - arrs = [_nx.array(a, ndmin=ndim) for a in arrs] + arrs = [array(a, ndmin=ndim, copy=False, subok=True) for a in arrs] return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim else: # We've 'bottomed out' - return _nx.array(arrays, ndmin=depth), depth + return array(arrays, ndmin=depth, copy=False, subok=True), depth def block(arrays): From 95adb77009483d76cc30db4f9524c6797b1d4e51 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 11 Sep 2017 00:36:50 +0100 Subject: [PATCH 04/23] Add empty list comment to block depth check --- numpy/core/shape_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 451ed4319b86..d1a37a1641f1 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -394,6 +394,7 @@ def format_index(index): ) return first_index elif isinstance(arrays, list) and len(arrays) == 0: + # We've 'bottomed out' on an empty list return index + [None] else: # We've 'bottomed out' From 07a3f43cd6406433c2132e7bd14ad43b23677ecd Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 11 Sep 2017 11:10:53 +0100 Subject: [PATCH 05/23] Use strict type checking (not isinstance) --- numpy/core/shape_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index d1a37a1641f1..cc164e8da2ab 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -369,7 +369,7 @@ def _block_check_depths_match(arrays, index=[]): def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) return 'arrays' + idx_str - if isinstance(arrays, tuple): + if type(arrays) is tuple: raise TypeError( '{} is a tuple. ' 'Only lists can be used to arrange blocks, and np.block does ' @@ -377,7 +377,7 @@ def format_index(index): format_index(index) ) ) - elif isinstance(arrays, list) and len(arrays) > 0: + elif type(arrays) is list and len(arrays) > 0: indexes = [_block_check_depths_match(arr, index + [i]) for i, arr in enumerate(arrays)] @@ -393,7 +393,7 @@ def format_index(index): ) ) return first_index - elif isinstance(arrays, list) and len(arrays) == 0: + elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list return index + [None] else: @@ -402,7 +402,7 @@ def format_index(index): def _block(arrays, depth=0): - if isinstance(arrays, list): + if type(arrays) is list: if len(arrays) == 0: raise ValueError('Lists cannot be empty') arrs, list_ndims = zip(*(_block(arr, depth+1) for arr in arrays)) From 19fc68c320282553020ca8f70761244f26cea8b0 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 11 Sep 2017 11:16:37 +0100 Subject: [PATCH 06/23] Re-add tuple type-check comment --- numpy/core/shape_base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index cc164e8da2ab..56b58e79c6b3 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -370,6 +370,11 @@ def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) return 'arrays' + idx_str if type(arrays) is tuple: + # not strictly necessary, but saves us from: + # - more than one way to do things - no point treating tuples like + # lists + # - horribly confusing behaviour that results when tuples are + # treated like ndarray raise TypeError( '{} is a tuple. ' 'Only lists can be used to arrange blocks, and np.block does ' From 997ac2cfbe3edba968e1ee140763b53b88c3afca Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 11 Sep 2017 11:21:59 +0100 Subject: [PATCH 07/23] Re-add `atleast_nd` function. --- numpy/core/shape_base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 56b58e79c6b3..bc5d5a991570 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -407,6 +407,11 @@ def format_index(index): def _block(arrays, depth=0): + def atleast_nd(a, ndim): + # Ensures `a` has at least `ndim` dimensions by prepending + # ones to `a.shape` as necessary + return array(a, ndmin=ndim, copy=False, subok=True) + if type(arrays) is list: if len(arrays) == 0: raise ValueError('Lists cannot be empty') @@ -414,11 +419,11 @@ def _block(arrays, depth=0): list_ndim = list_ndims[0] arr_ndim = max(arr.ndim for arr in arrs) ndim = max(list_ndim, arr_ndim) - arrs = [array(a, ndmin=ndim, copy=False, subok=True) for a in arrs] + arrs = [atleast_nd(a, ndim) for a in arrs] return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim else: # We've 'bottomed out' - return array(arrays, ndmin=depth, copy=False, subok=True), depth + return atleast_nd(arrays, depth), depth def block(arrays): From 7eb104463f192f50d184c3217a0c0febe3a88c94 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Tue, 19 Sep 2017 00:14:39 +0100 Subject: [PATCH 08/23] Add detailed comment to _block_check_depths_match --- numpy/core/shape_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index bc5d5a991570..ad2a68f202ef 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -366,6 +366,12 @@ def stack(arrays, axis=0, out=None): def _block_check_depths_match(arrays, index=[]): + # Recursive function checking that the depths of nested lists in `arrays` + # all match. Mismatch raises a ValueError as described in the block + # docstring below. + # The entire index (rather than just the depth) is calculated for each + # innermost list, in case an error needs to be raised, so that the index + # of the offending list can be printed as part of the error. def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) return 'arrays' + idx_str From ff7f72690f810d00c13d071fc5c4205c6a21cccf Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Wed, 20 Sep 2017 17:37:59 +0100 Subject: [PATCH 09/23] Extend comments _block_check_depths_match --- numpy/core/shape_base.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index ad2a68f202ef..f28c8f1b4293 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -365,13 +365,23 @@ def stack(arrays, axis=0, out=None): return _nx.concatenate(expanded_arrays, axis=axis, out=out) -def _block_check_depths_match(arrays, index=[]): - # Recursive function checking that the depths of nested lists in `arrays` - # all match. Mismatch raises a ValueError as described in the block - # docstring below. - # The entire index (rather than just the depth) is calculated for each - # innermost list, in case an error needs to be raised, so that the index - # of the offending list can be printed as part of the error. +def _block_check_depths_match(arrays, parent_index=[]): + """ + Recursive function checking that the depths of nested lists in `arrays` + all match. Mismatch raises a ValueError as described in the block + docstring below. + + The entire index (rather than just the depth) needs to be calculated + for each innermost list, in case an error needs to be raised, so that + the index of the offending list can be printed as part of the error. + + The parameter `parent_index` is the full index of `arrays` within the + nested lists passed to _block_check_depths_match at the top of the + recursion. + The return value is the full index of an element (specifically the + first element) from the bottom of the nesting in `arrays`. An empty + list at the bottom of the nesting is represented by a `None` index. + """ def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) return 'arrays' + idx_str @@ -385,11 +395,11 @@ def format_index(index): '{} is a tuple. ' 'Only lists can be used to arrange blocks, and np.block does ' 'not allow implicit conversion from tuple to ndarray.'.format( - format_index(index) + format_index(parent_index) ) ) elif type(arrays) is list and len(arrays) > 0: - indexes = [_block_check_depths_match(arr, index + [i]) + indexes = [_block_check_depths_match(arr, parent_index + [i]) for i, arr in enumerate(arrays)] first_index = indexes[0] @@ -406,10 +416,10 @@ def format_index(index): return first_index elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list - return index + [None] + return parent_index + [None] else: - # We've 'bottomed out' - return index + # We've 'bottomed out' - arrays is either a scalar or an array + return parent_index def _block(arrays, depth=0): @@ -428,7 +438,7 @@ def atleast_nd(a, ndim): arrs = [atleast_nd(a, ndim) for a in arrs] return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim else: - # We've 'bottomed out' + # We've 'bottomed out' - arrays is either a scalar or an array return atleast_nd(arrays, depth), depth From 6ecd2b476203b36b369b8dd4ee0725a0625d593b Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Sat, 23 Sep 2017 09:25:22 +0100 Subject: [PATCH 10/23] Try not recomputing list_ndim --- numpy/core/shape_base.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index f28c8f1b4293..d909dfe50b71 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -8,6 +8,7 @@ from .numeric import array, asanyarray, newaxis from .multiarray import normalize_axis_index + def atleast_1d(*arys): """ Convert inputs to arrays with at least one dimension. @@ -422,24 +423,26 @@ def format_index(index): return parent_index -def _block(arrays, depth=0): +def _block(arrays, list_ndim): def atleast_nd(a, ndim): # Ensures `a` has at least `ndim` dimensions by prepending # ones to `a.shape` as necessary return array(a, ndmin=ndim, copy=False, subok=True) - if type(arrays) is list: - if len(arrays) == 0: - raise ValueError('Lists cannot be empty') - arrs, list_ndims = zip(*(_block(arr, depth+1) for arr in arrays)) - list_ndim = list_ndims[0] - arr_ndim = max(arr.ndim for arr in arrs) - ndim = max(list_ndim, arr_ndim) - arrs = [atleast_nd(a, ndim) for a in arrs] - return _nx.concatenate(arrs, axis=depth+ndim-list_ndim), list_ndim - else: - # We've 'bottomed out' - arrays is either a scalar or an array - return atleast_nd(arrays, depth), depth + def block_recursion(arrays, depth=0): + if type(arrays) is list: + if len(arrays) == 0: + raise ValueError('Lists cannot be empty') + arrs = [block_recursion(arr, depth+1) for arr in arrays] + arr_ndim = max(arr.ndim for arr in arrs) + ndim = max(list_ndim, arr_ndim) + arrs = [atleast_nd(a, ndim) for a in arrs] + return _nx.concatenate(arrs, axis=depth+ndim-list_ndim) + else: + # We've 'bottomed out' - arrays is either a scalar or an array + return atleast_nd(arrays, depth) + + return block_recursion(arrays) def block(arrays): @@ -590,5 +593,5 @@ def block(arrays): """ - _block_check_depths_match(arrays) - return _block(arrays)[0] + list_ndim = len(_block_check_depths_match(arrays)) + return _block(arrays, list_ndim) From 1211b70bbd2862dc41c2f25cb7c85493f10ccd60 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 25 Sep 2017 10:40:57 +0100 Subject: [PATCH 11/23] Slight simplification to logic --- numpy/core/shape_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index d909dfe50b71..306754a59e7b 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -435,12 +435,11 @@ def block_recursion(arrays, depth=0): raise ValueError('Lists cannot be empty') arrs = [block_recursion(arr, depth+1) for arr in arrays] arr_ndim = max(arr.ndim for arr in arrs) - ndim = max(list_ndim, arr_ndim) - arrs = [atleast_nd(a, ndim) for a in arrs] - return _nx.concatenate(arrs, axis=depth+ndim-list_ndim) + arrs = [atleast_nd(a, arr_ndim) for a in arrs] + return _nx.concatenate(arrs, axis=depth+max(0, arr_ndim-list_ndim)) else: # We've 'bottomed out' - arrays is either a scalar or an array - return atleast_nd(arrays, depth) + return atleast_nd(arrays, list_ndim) return block_recursion(arrays) From ffa6cf66b03dab41ae843f3b8117a279fb2ef5c2 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 25 Sep 2017 11:26:07 +0100 Subject: [PATCH 12/23] Add two tests for different arr_ndims --- numpy/core/tests/test_shape_base.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 5c1e569b7d9a..deb2a407d78c 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -560,6 +560,28 @@ def test_tuple(self): assert_raises_regex(TypeError, 'tuple', np.block, ([1, 2], [3, 4])) assert_raises_regex(TypeError, 'tuple', np.block, [(1, 2), (3, 4)]) + def test_different_ndims(self): + a = 1. + b = 2 * np.ones((1, 2)) + c = 3 * np.ones((1, 1, 3)) + + result = np.block([a, b, c]) + expected = np.array([[[1., 2., 2., 3., 3., 3.]]]) + + assert_equal(result, expected) + + def test_different_ndims_depths(self): + a = 1. + b = 2 * np.ones((1, 2)) + c = 3 * np.ones((1, 2, 3)) + + result = np.block([[a, b], [c]]) + expected = np.array([[[1., 2., 2.], + [3., 3., 3.], + [3., 3., 3.]]]) + + assert_equal(result, expected) + if __name__ == "__main__": run_module_suite() From 3ed6936b82cb444e5516bda705701245d4fa4b4d Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 25 Sep 2017 11:41:12 +0100 Subject: [PATCH 13/23] Simplify further - matching docstring logic --- numpy/core/shape_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 306754a59e7b..95cda658db4f 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -436,7 +436,7 @@ def block_recursion(arrays, depth=0): arrs = [block_recursion(arr, depth+1) for arr in arrays] arr_ndim = max(arr.ndim for arr in arrs) arrs = [atleast_nd(a, arr_ndim) for a in arrs] - return _nx.concatenate(arrs, axis=depth+max(0, arr_ndim-list_ndim)) + return _nx.concatenate(arrs, axis=depth-list_ndim) else: # We've 'bottomed out' - arrays is either a scalar or an array return atleast_nd(arrays, list_ndim) From 5f9f1faf6fc0bacd737bb191b2ae8c054fe7c7ae Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 25 Sep 2017 11:59:03 +0100 Subject: [PATCH 14/23] Rename list_ndim to max_depth --- numpy/core/shape_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 95cda658db4f..73639f286222 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -423,7 +423,7 @@ def format_index(index): return parent_index -def _block(arrays, list_ndim): +def _block(arrays, max_depth): def atleast_nd(a, ndim): # Ensures `a` has at least `ndim` dimensions by prepending # ones to `a.shape` as necessary @@ -436,10 +436,11 @@ def block_recursion(arrays, depth=0): arrs = [block_recursion(arr, depth+1) for arr in arrays] arr_ndim = max(arr.ndim for arr in arrs) arrs = [atleast_nd(a, arr_ndim) for a in arrs] - return _nx.concatenate(arrs, axis=depth-list_ndim) + return _nx.concatenate(arrs, axis=-(max_depth-depth)) else: # We've 'bottomed out' - arrays is either a scalar or an array - return atleast_nd(arrays, list_ndim) + # depth == max_depth + return atleast_nd(arrays, max_depth) return block_recursion(arrays) From 8a83a5fcf8580fbcb3caf3ab3d5971876d9da959 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Mon, 25 Sep 2017 12:11:40 +0100 Subject: [PATCH 15/23] rm extra line from near top of shape_base --- numpy/core/shape_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 73639f286222..22a8ac304c8b 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -8,7 +8,6 @@ from .numeric import array, asanyarray, newaxis from .multiarray import normalize_axis_index - def atleast_1d(*arys): """ Convert inputs to arrays with at least one dimension. From 5a0557ae36dbba08ce3374f56b8d1502913123e4 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 12 Oct 2017 10:30:21 +0100 Subject: [PATCH 16/23] Pre-calculate max array ndim --- numpy/core/shape_base.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 22a8ac304c8b..c65849752cb7 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -399,8 +399,8 @@ def format_index(index): ) ) elif type(arrays) is list and len(arrays) > 0: - indexes = [_block_check_depths_match(arr, parent_index + [i]) - for i, arr in enumerate(arrays)] + indexes, arr_ndims = zip(*[_block_check_depths_match(arr, parent_index + [i]) + for i, arr in enumerate(arrays)]) first_index = indexes[0] for i, index in enumerate(indexes): @@ -413,16 +413,16 @@ def format_index(index): format_index(index) ) ) - return first_index + return first_index, max(arr_ndims) elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list - return parent_index + [None] + return parent_index + [None], _nx.ndim(arrays) else: # We've 'bottomed out' - arrays is either a scalar or an array - return parent_index + return parent_index, _nx.ndim(arrays) -def _block(arrays, max_depth): +def _block(arrays, max_depth, max_ndim): def atleast_nd(a, ndim): # Ensures `a` has at least `ndim` dimensions by prepending # ones to `a.shape` as necessary @@ -433,13 +433,11 @@ def block_recursion(arrays, depth=0): if len(arrays) == 0: raise ValueError('Lists cannot be empty') arrs = [block_recursion(arr, depth+1) for arr in arrays] - arr_ndim = max(arr.ndim for arr in arrs) - arrs = [atleast_nd(a, arr_ndim) for a in arrs] return _nx.concatenate(arrs, axis=-(max_depth-depth)) else: # We've 'bottomed out' - arrays is either a scalar or an array # depth == max_depth - return atleast_nd(arrays, max_depth) + return atleast_nd(arrays, max(max_depth, max_ndim)) return block_recursion(arrays) @@ -592,5 +590,6 @@ def block(arrays): """ - list_ndim = len(_block_check_depths_match(arrays)) - return _block(arrays, list_ndim) + bottom_index, arr_ndim = _block_check_depths_match(arrays) + list_ndim = len(bottom_index) + return _block(arrays, list_ndim, arr_ndim) From c2b5be5e75bd665e02c97b3d03f559dfde485fbb Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 12 Oct 2017 13:54:42 +0100 Subject: [PATCH 17/23] Further slight simplifications --- numpy/core/shape_base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index c65849752cb7..732ea30ae942 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -399,8 +399,8 @@ def format_index(index): ) ) elif type(arrays) is list and len(arrays) > 0: - indexes, arr_ndims = zip(*[_block_check_depths_match(arr, parent_index + [i]) - for i, arr in enumerate(arrays)]) + indexes, arr_ndims = zip(*(_block_check_depths_match(arr, parent_index + [i]) + for i, arr in enumerate(arrays))) first_index = indexes[0] for i, index in enumerate(indexes): @@ -422,14 +422,14 @@ def format_index(index): return parent_index, _nx.ndim(arrays) -def _block(arrays, max_depth, max_ndim): +def _block(arrays, max_depth, result_ndim): def atleast_nd(a, ndim): # Ensures `a` has at least `ndim` dimensions by prepending # ones to `a.shape` as necessary return array(a, ndmin=ndim, copy=False, subok=True) def block_recursion(arrays, depth=0): - if type(arrays) is list: + if depth < max_depth: if len(arrays) == 0: raise ValueError('Lists cannot be empty') arrs = [block_recursion(arr, depth+1) for arr in arrays] @@ -437,7 +437,7 @@ def block_recursion(arrays, depth=0): else: # We've 'bottomed out' - arrays is either a scalar or an array # depth == max_depth - return atleast_nd(arrays, max(max_depth, max_ndim)) + return atleast_nd(arrays, result_ndim) return block_recursion(arrays) @@ -592,4 +592,4 @@ def block(arrays): """ bottom_index, arr_ndim = _block_check_depths_match(arrays) list_ndim = len(bottom_index) - return _block(arrays, list_ndim, arr_ndim) + return _block(arrays, list_ndim, max(arr_ndim, list_ndim)) From bd6729d0d1a6004360f44524a477524ccdc96f17 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 12 Oct 2017 14:13:34 +0100 Subject: [PATCH 18/23] Update block docstrings --- numpy/core/shape_base.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 732ea30ae942..217631cd3e7e 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -378,9 +378,12 @@ def _block_check_depths_match(arrays, parent_index=[]): The parameter `parent_index` is the full index of `arrays` within the nested lists passed to _block_check_depths_match at the top of the recursion. - The return value is the full index of an element (specifically the - first element) from the bottom of the nesting in `arrays`. An empty - list at the bottom of the nesting is represented by a `None` index. + The return value is a pair. The first item returned is the full index + of an element (specifically the first element) from the bottom of the + nesting in `arrays`. An empty list at the bottom of the nesting is + represented by a `None` index. + The second item is the maximum of the ndims of the arrays nested in + `arrays`. """ def format_index(index): idx_str = ''.join('[{}]'.format(i) for i in index if i is not None) @@ -423,6 +426,13 @@ def format_index(index): def _block(arrays, max_depth, result_ndim): + """ + Internal implementation of block. `arrays` is the argument passed to + block. `max_depth` is the depth of nested lists within `arrays` and + `result_ndim` is the greatest of the dimensions of the arrays in + `arrays` and the depth of the lists in `arrays` (see block docstring + for details). + """ def atleast_nd(a, ndim): # Ensures `a` has at least `ndim` dimensions by prepending # ones to `a.shape` as necessary @@ -436,7 +446,7 @@ def block_recursion(arrays, depth=0): return _nx.concatenate(arrs, axis=-(max_depth-depth)) else: # We've 'bottomed out' - arrays is either a scalar or an array - # depth == max_depth + # type(arrays) is not list return atleast_nd(arrays, result_ndim) return block_recursion(arrays) From ad278f3bfea97636b18c8d699babdd02aa3cac5c Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 12 Oct 2017 14:44:03 +0100 Subject: [PATCH 19/23] Fix python 3.4 sequence error --- numpy/core/shape_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 217631cd3e7e..23e2ef57f55a 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -402,8 +402,8 @@ def format_index(index): ) ) elif type(arrays) is list and len(arrays) > 0: - indexes, arr_ndims = zip(*(_block_check_depths_match(arr, parent_index + [i]) - for i, arr in enumerate(arrays))) + indexes, arr_ndims = zip(*[_block_check_depths_match(arr, parent_index + [i]) + for i, arr in enumerate(arrays)]) first_index = indexes[0] for i, index in enumerate(indexes): From 2c1734b192e0d5cb3973b76f9319b18be2a44697 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 26 Oct 2017 11:56:45 +0100 Subject: [PATCH 20/23] Correct empty list ndim --- numpy/core/shape_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 23e2ef57f55a..b7fe71310059 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -419,7 +419,7 @@ def format_index(index): return first_index, max(arr_ndims) elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list - return parent_index + [None], _nx.ndim(arrays) + return parent_index + [None], 0 else: # We've 'bottomed out' - arrays is either a scalar or an array return parent_index, _nx.ndim(arrays) From eaddf3935ab94def7439f8674e1444af2f12cfbb Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Wed, 1 Nov 2017 13:16:24 +0000 Subject: [PATCH 21/23] Avoid using zip(*...) syntax --- numpy/core/shape_base.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index b7fe71310059..29765d0783ff 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -402,11 +402,13 @@ def format_index(index): ) ) elif type(arrays) is list and len(arrays) > 0: - indexes, arr_ndims = zip(*[_block_check_depths_match(arr, parent_index + [i]) - for i, arr in enumerate(arrays)]) + idxs_ndims = (_block_check_depths_match(arr, parent_index + [i]) + for i, arr in enumerate(arrays)) - first_index = indexes[0] - for i, index in enumerate(indexes): + first_index, max_arr_ndim = idxs_ndims.__next__() + for i, (index, ndim) in enumerate(idxs_ndims, 1): + if ndim > max_arr_ndim: + max_arr_ndim = ndim if len(index) != len(first_index): raise ValueError( "List depths are mismatched. First element was at depth " @@ -416,7 +418,7 @@ def format_index(index): format_index(index) ) ) - return first_index, max(arr_ndims) + return first_index, max_arr_ndim elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list return parent_index + [None], 0 From a5cbc93168e08f7a083a7df174a44314c1af9bd4 Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Wed, 1 Nov 2017 13:25:42 +0000 Subject: [PATCH 22/23] Use builtin next method --- numpy/core/shape_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 29765d0783ff..ca0faca97ceb 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -405,7 +405,7 @@ def format_index(index): idxs_ndims = (_block_check_depths_match(arr, parent_index + [i]) for i, arr in enumerate(arrays)) - first_index, max_arr_ndim = idxs_ndims.__next__() + first_index, max_arr_ndim = next(idxs_ndims) for i, (index, ndim) in enumerate(idxs_ndims, 1): if ndim > max_arr_ndim: max_arr_ndim = ndim From a691f2deb055f4ebfd60f742d816980c6893871f Mon Sep 17 00:00:00 2001 From: Jamie Townsend Date: Thu, 9 Nov 2017 12:56:10 +0000 Subject: [PATCH 23/23] Rm unnecessary enumerate --- numpy/core/shape_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index ca0faca97ceb..8a047fddab1c 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -406,7 +406,7 @@ def format_index(index): for i, arr in enumerate(arrays)) first_index, max_arr_ndim = next(idxs_ndims) - for i, (index, ndim) in enumerate(idxs_ndims, 1): + for index, ndim in idxs_ndims: if ndim > max_arr_ndim: max_arr_ndim = ndim if len(index) != len(first_index):