Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit 05118ea

Browse files
kozlov-alexeykozlov-alexey
committed
Merge branch 'master' into feature/migrate_numba_0.54
2 parents 17ab7fe + 8049dba commit 05118ea

26 files changed

+2623
-77
lines changed

‎README.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ Building on Linux with setuptools
8585

8686
export PYVER=<3.6 or 3.7>
8787
export NUMPYVER=<1.16 or 1.17>
88-
conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.52 pandas=1.2.0 pyarrow=2.0.0 gcc_linux-64 gxx_linux-64
88+
conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.53.1 pandas=1.2.0 pyarrow=4.0.1 gcc_linux-64 gxx_linux-64
8989
source activate sdc-env
9090
git clone https://github.com/IntelPython/sdc.git
9191
cd sdc
@@ -123,7 +123,7 @@ Building on Windows with setuptools
123123

124124
set PYVER=<3.6 or 3.7>
125125
set NUMPYVER=<1.16 or 1.17>
126-
conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.52 pandas=1.2.0 pyarrow=2.0.0
126+
conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.53.1 pandas=1.2.0 pyarrow=4.0.1
127127
conda activate sdc-env
128128
set INCLUDE=%INCLUDE%;%CONDA_PREFIX%\Library\include
129129
set LIB=%LIB%;%CONDA_PREFIX%\Library\lib

‎conda-recipe/meta.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{% set NUMBA_VERSION = ">0.53.1" %}
22
{% set PANDAS_VERSION = "==1.2.0" %}
3-
{% set PYARROW_VERSION = "==2.0.0" %}
3+
{% set PYARROW_VERSION = "==4.0.1" %}
44

55
package:
66
name: sdc

‎docs/source/getting_started.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,14 @@ Distribution includes Intel SDC for Python 3.6 and 3.7 for Windows and Linux pla
4141
Intel SDC conda package can be installed using the steps below:
4242
::
4343

44-
> conda create -n sdc_env python=<3.7 or 3.6> pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
44+
> conda create -n sdc_env python=<3.7 or 3.6> pyarrow=4.0.1 pandas=1.2.0 -c anaconda -c conda-forge
4545
> conda activate sdc_env
4646
> conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
4747

4848
Intel SDC wheel package can be installed using the steps below:
4949
::
5050

51-
> conda create -n sdc_env python=<3.7 or 3.6> pip pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
51+
> conda create -n sdc_env python=<3.7 or 3.6> pip pyarrow=4.0.1 pandas=1.2.0 -c anaconda -c conda-forge
5252
> conda activate sdc_env
5353
> pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
5454

‎requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
numpy>=1.16
22
pandas==1.2.0
3-
pyarrow==2.0.0
4-
numba>0.53.1
3+
pyarrow==4.0.1
4+
numba==0.54.0
55
tbb
66
tbb-devel

‎sdc/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949

5050
import sdc.extensions.indexes.range_index_ext
5151
import sdc.extensions.indexes.int64_index_ext
52+
import sdc.extensions.indexes.multi_index_ext
5253

5354
import sdc.extensions.sdc_hashmap_ext
5455

‎sdc/datatypes/hpat_pandas_series_functions.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4574,7 +4574,7 @@ def _series_operator_add_str_impl(self, other):
45744574
else:
45754575
indexes_join_res = sdc_indexes_join_outer(left_index, right_index)
45764576

4577-
# FIXME_Numba#XXXX: remove sdc_fix_indexes_join call at all when issue is fixed
4577+
# FIXME_Numba#6686: remove sdc_fix_indexes_join call at all when issue is fixed
45784578
joined_index, left_indexer, right_indexer = sdc_fix_indexes_join(*indexes_join_res)
45794579
result_size = len(joined_index)
45804580
result_nan_mask = numpy.zeros(result_size, dtype=numpy.bool_)
@@ -4692,7 +4692,7 @@ def _series_operator_mul_common_impl(self, other):
46924692
else:
46934693
indexes_join_res = sdc_indexes_join_outer(left_index, right_index)
46944694

4695-
# FIXME_Numba#XXXX: remove sdc_fix_indexes_join call at all when issue is fixed
4695+
# FIXME_Numba#6686: remove sdc_fix_indexes_join call at all when issue is fixed
46964696
joined_index, left_indexer, right_indexer = sdc_fix_indexes_join(*indexes_join_res)
46974697
str_series_operand = self if self_is_string_series == True else other # noqa
46984698
str_series_indexer = left_indexer if self_is_string_series == True else right_indexer # noqa

‎sdc/datatypes/indexes/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@
3030
from .positional_index_type import PositionalIndexType
3131
from .empty_index_type import EmptyIndexType
3232
from .int64_index_type import Int64IndexType
33+
from .multi_index_type import MultiIndexType

‎sdc/datatypes/indexes/multi_index_type.py

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# -*- coding: utf-8 -*-
2+
# *****************************************************************************
3+
# Copyright (c) 2021, Intel Corporation All rights reserved.
4+
#
5+
# Redistribution and use in source and binary forms, with or without
6+
# modification, are permitted provided that the following conditions are met:
7+
#
8+
# Redistributions of source code must retain the above copyright notice,
9+
# this list of conditions and the following disclaimer.
10+
#
11+
# Redistributions in binary form must reproduce the above copyright notice,
12+
# this list of conditions and the following disclaimer in the documentation
13+
# and/or other materials provided with the distribution.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
25+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
# *****************************************************************************
27+
28+
from numba import types
29+
from numba.extending import (
30+
models,
31+
register_model,
32+
make_attribute_wrapper,
33+
)
34+
35+
36+
class MultiIndexIteratorType(types.SimpleIteratorType):
37+
def __init__(self, iterable):
38+
self.parent = iterable
39+
yield_type = iterable.dtype
40+
name = "iter[{}->{}],{}".format(
41+
iterable, yield_type, iterable.name
42+
)
43+
super(MultiIndexIteratorType, self).__init__(name, yield_type)
44+
45+
46+
@register_model(MultiIndexIteratorType)
47+
class MultiIndexIterModel(models.StructModel):
48+
def __init__(self, dmm, fe_type):
49+
members = [
50+
('parent', fe_type.parent), # reference to the index object
51+
('state', types.CPointer(types.int64)), # iterator state (i.e. counter)
52+
]
53+
super(MultiIndexIterModel, self).__init__(dmm, fe_type, members)
54+
55+
56+
class MultiIndexType(types.IterableType):
57+
58+
def __init__(self, levels, codes, is_named=False):
59+
self.levels = levels
60+
self.codes = codes
61+
self.is_named = is_named
62+
super(MultiIndexType, self).__init__(
63+
name='MultiIndexType({}, {}, {})'.format(levels, codes, is_named))
64+
65+
@property
66+
def iterator_type(self):
67+
return MultiIndexIteratorType(self).iterator_type
68+
69+
@property
70+
def dtype(self):
71+
nlevels = len(self.levels)
72+
levels_types = [self.levels.dtype] * nlevels if isinstance(self.levels, types.UniTuple) else self.levels
73+
return types.Tuple.from_types([level.dtype for level in levels_types])
74+
75+
@property
76+
def nlevels(self):
77+
return len(self.levels)
78+
79+
@property
80+
def levels_types(self):
81+
if isinstance(self.levels, types.UniTuple):
82+
return [self.levels.dtype] * self.levels.count
83+
84+
return self.levels
85+
86+
@property
87+
def codes_types(self):
88+
if isinstance(self.codes, types.UniTuple):
89+
return [self.codes.dtype] * self.codes.count
90+
91+
return self.codes
92+
93+
94+
@register_model(MultiIndexType)
95+
class MultiIndexModel(models.StructModel):
96+
def __init__(self, dmm, fe_type):
97+
98+
levels_type = fe_type.levels
99+
codes_type = fe_type.codes
100+
name_type = types.unicode_type if fe_type.is_named else types.none # TO-DO: change to types.Optional
101+
members = [
102+
('levels', levels_type),
103+
('codes', codes_type),
104+
('name', name_type),
105+
]
106+
models.StructModel.__init__(self, dmm, fe_type, members)
107+
108+
109+
make_attribute_wrapper(MultiIndexType, 'levels', '_levels')
110+
make_attribute_wrapper(MultiIndexType, 'codes', '_codes')
111+
make_attribute_wrapper(MultiIndexType, 'name', '_name')

‎sdc/datatypes/sdc_typeref.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# *****************************************************************************
2+
# Copyright (c) 2021, Intel Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
#
10+
# Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
18+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
# *****************************************************************************
26+
27+
import pandas as pd
28+
29+
from numba.core import types
30+
from numba.extending import (models, register_model, )
31+
from numba.core.typing.templates import infer_global
32+
33+
from sdc.extensions.sdc_hashmap_type import ConcurrentDict, ConcurrentDictType
34+
from sdc.datatypes.indexes import MultiIndexType
35+
36+
37+
# FIXME_Numba#6781: due to overlapping of overload_methods for Numba TypeRef
38+
# we have to use our new SdcTypeRef to type objects created from types.Type
39+
# (i.e. ConcurrentDict meta-type). This should be removed once it's fixed.
40+
def sdc_make_new_typeref_class():
41+
class SdcTypeRef(types.Dummy):
42+
"""Reference to a type.
43+
44+
Used when a type is passed as a value.
45+
"""
46+
def __init__(self, instance_type):
47+
self.instance_type = instance_type
48+
super(SdcTypeRef, self).__init__('sdc_typeref[{}]'.format(self.instance_type))
49+
50+
@register_model(SdcTypeRef)
51+
class SdcTypeRefModel(models.OpaqueModel):
52+
def __init__(self, dmm, fe_type):
53+
54+
models.OpaqueModel.__init__(self, dmm, fe_type)
55+
56+
return SdcTypeRef
57+
58+
59+
ConcurrentDictTypeRef = sdc_make_new_typeref_class()
60+
MultiIndexTypeRef = sdc_make_new_typeref_class()
61+
62+
infer_global(ConcurrentDict, ConcurrentDictTypeRef(ConcurrentDictType))
63+
infer_global(pd.MultiIndex, MultiIndexTypeRef(MultiIndexType))

0 commit comments

Comments
 (0)