Skip to content

Commit cee5a38

Browse files
lestevelesteve
authored
[MRG] Update joblib to 0.11 (#8492)
Use pip rather than easy_install in copy_joblib.sh. Also need to remove joblib/testing.py to avoid pytest dependency.
1 parent 5210f81 commit cee5a38

17 files changed

+511
-400
lines changed

‎sklearn/externals/copy_joblib.sh

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,12 @@
11
#!/bin/sh
22
# Script to do a local install of joblib
33
export LC_ALL=C
4-
rm -rf tmp joblib
5-
PYTHON_VERSION=$(python -c 'import sys; print("{0[0]}.{0[1]}".format(sys.version_info))')
6-
SITE_PACKAGES="$PWD/tmp/lib/python$PYTHON_VERSION/site-packages"
4+
INSTALL_FOLDER=tmp/joblib_install
5+
rm -rf joblib $INSTALL_FOLDER
6+
pip install joblib --target $INSTALL_FOLDER
7+
cp -r $INSTALL_FOLDER/joblib .
8+
rm -rf $INSTALL_FOLDER
79

8-
mkdir -p $SITE_PACKAGES
9-
mkdir -p tmp/bin
10-
export PYTHONPATH="$SITE_PACKAGES"
11-
easy_install -Zeab tmp joblib
12-
13-
cd tmp/joblib/
14-
python setup.py install --prefix $OLDPWD/tmp
15-
cd $OLDPWD
16-
cp -r $SITE_PACKAGES/joblib-*.egg/joblib .
17-
rm -rf tmp
1810
# Needed to rewrite the doctests
1911
# Note: BSD sed -i needs an argument unders OSX
2012
# so first renaming to .bak and then deleting backup files
@@ -25,4 +17,6 @@ find joblib -name "*.bak" | xargs rm
2517
# joblib is already tested on its own CI infrastructure upstream.
2618
rm -r joblib/test
2719

28-
chmod -x joblib/*.py
20+
# Remove joblib/testing.py which is only used in tests and has a
21+
# pytest dependency (needed until we drop nose)
22+
rm joblib/testing.py

‎sklearn/externals/joblib/__init__.py

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
1-
""" Joblib is a set of tools to provide **lightweight pipelining in
1+
"""Joblib is a set of tools to provide **lightweight pipelining in
22
Python**. In particular, joblib offers:
33
4-
1. transparent disk-caching of the output values and lazy re-evaluation
5-
(memoize pattern)
4+
1. transparent disk-caching of the output values and lazy re-evaluation
5+
(memoize pattern)
66
7-
2. easy simple parallel computing
7+
2. easy simple parallel computing
88
9-
3. logging and tracing of the execution
9+
3. logging and tracing of the execution
1010
1111
Joblib is optimized to be **fast** and **robust** in particular on large
1212
data and has specific optimizations for `numpy` arrays. It is
1313
**BSD-licensed**.
1414
1515
16-
============================== ============================================
17-
**User documentation**: http://pythonhosted.org/joblib
16+
========================= ================================================
17+
**User documentation:** http://pythonhosted.org/joblib
1818
19-
**Download packages**: http://pypi.python.org/pypi/joblib#downloads
19+
**Download packages:** http://pypi.python.org/pypi/joblib#downloads
2020
21-
**Source code**: http://github.com/joblib/joblib
21+
**Source code:** http://github.com/joblib/joblib
2222
23-
**Report issues**: http://github.com/joblib/joblib/issues
24-
============================== ============================================
23+
**Report issues:** http://github.com/joblib/joblib/issues
24+
========================= ================================================
2525
2626
2727
Vision
@@ -115,8 +115,7 @@
115115
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
116116
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
117117
#
118-
119-
__version__ = '0.10.3'
118+
__version__ = '0.11'
120119

121120

122121
from .memory import Memory, MemorizedResult

‎sklearn/externals/joblib/_compat.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import sys
55

66
PY3_OR_LATER = sys.version_info[0] >= 3
7-
PY26 = sys.version_info[:2] == (2, 6)
87
PY27 = sys.version_info[:2] == (2, 7)
98

109
try:

‎sklearn/externals/joblib/_memory_helpers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,4 +102,4 @@ def open_py_source(filename):
102102
buffer.seek(0)
103103
text = TextIOWrapper(buffer, encoding, line_buffering=True)
104104
text.mode = 'r'
105-
return text
105+
return text

‎sklearn/externals/joblib/_parallel_backends.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
class ParallelBackendBase(with_metaclass(ABCMeta)):
2222
"""Helper abc which defines all methods a ParallelBackend must implement"""
2323

24+
supports_timeout = False
25+
2426
@abstractmethod
2527
def effective_n_jobs(self, n_jobs):
2628
"""Determine the number of jobs that can actually run in parallel
@@ -236,6 +238,8 @@ class ThreadingBackend(PoolManagerMixin, ParallelBackendBase):
236238
"with nogil" block or an expensive call to a library such as NumPy).
237239
"""
238240

241+
supports_timeout = True
242+
239243
def configure(self, n_jobs=1, parallel=None, **backend_args):
240244
"""Build a process or thread pool and return the number of workers"""
241245
n_jobs = self.effective_n_jobs(n_jobs)
@@ -259,6 +263,8 @@ class MultiprocessingBackend(PoolManagerMixin, AutoBatchingMixin,
259263
# Environment variables to protect against bad situations when nesting
260264
JOBLIB_SPAWNED_PROCESS = "__JOBLIB_SPAWNED_PARALLEL__"
261265

266+
supports_timeout = True
267+
262268
def effective_n_jobs(self, n_jobs):
263269
"""Determine the number of jobs which are going to run in parallel.
264270
@@ -277,10 +283,10 @@ def effective_n_jobs(self, n_jobs):
277283
stacklevel=3)
278284
return 1
279285

280-
elif threading.current_thread().name != 'MainThread':
286+
if not isinstance(threading.current_thread(), threading._MainThread):
281287
# Prevent posix fork inside in non-main posix threads
282288
warnings.warn(
283-
'Multiprocessing backed parallel loops cannot be nested'
289+
'Multiprocessing-backed parallel loops cannot be nested'
284290
' below threads, setting n_jobs=1',
285291
stacklevel=3)
286292
return 1

‎sklearn/externals/joblib/backports.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""
2+
Backports of fixes for joblib dependencies
3+
"""
4+
import os
5+
import time
6+
import ctypes
7+
import sys
8+
9+
from distutils.version import LooseVersion
10+
11+
try:
12+
import numpy as np
13+
14+
def make_memmap(filename, dtype='uint8', mode='r+', offset=0,
15+
shape=None, order='C'):
16+
"""Backport of numpy memmap offset fix.
17+
18+
See https://github.com/numpy/numpy/pull/8443 for more details.
19+
20+
The numpy fix will be available in numpy 1.13.
21+
"""
22+
mm = np.memmap(filename, dtype=dtype, mode=mode, offset=offset,
23+
shape=shape, order=order)
24+
if LooseVersion(np.__version__) < '1.13':
25+
mm.offset = offset
26+
return mm
27+
except ImportError:
28+
def make_memmap(filename, dtype='uint8', mode='r+', offset=0,
29+
shape=None, order='C'):
30+
raise NotImplementedError(
31+
"'joblib.backports.make_memmap' should not be used "
32+
'if numpy is not installed.')
33+
34+
35+
if os.name == 'nt':
36+
error_access_denied = 5
37+
try:
38+
from os import replace
39+
except ImportError:
40+
# Python 2.7
41+
def replace(src, dst):
42+
if not isinstance(src, unicode): # noqa
43+
src = unicode(src, sys.getfilesystemencoding()) # noqa
44+
if not isinstance(dst, unicode): # noqa
45+
dst = unicode(dst, sys.getfilesystemencoding()) # noqa
46+
47+
movefile_replace_existing = 0x1
48+
return_value = ctypes.windll.kernel32.MoveFileExW(
49+
src, dst, movefile_replace_existing)
50+
if return_value == 0:
51+
raise ctypes.WinError()
52+
53+
def concurrency_safe_rename(src, dst):
54+
"""Renames ``src`` into ``dst`` overwriting ``dst`` if it exists.
55+
56+
On Windows os.replace (or for Python 2.7 its implementation
57+
through MoveFileExW) can yield permission errors if executed by
58+
two different processes.
59+
"""
60+
max_sleep_time = 1
61+
total_sleep_time = 0
62+
sleep_time = 0.001
63+
while total_sleep_time < max_sleep_time:
64+
try:
65+
replace(src, dst)
66+
break
67+
except Exception as exc:
68+
if getattr(exc, 'winerror', None) == error_access_denied:
69+
time.sleep(sleep_time)
70+
total_sleep_time += sleep_time
71+
sleep_time *= 2
72+
else:
73+
raise
74+
else:
75+
raise
76+
else:
77+
try:
78+
from os import replace as concurrency_safe_rename
79+
except ImportError:
80+
from os import rename as concurrency_safe_rename # noqa

‎sklearn/externals/joblib/format_stack.py

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -135,15 +135,10 @@ def _fixed_getframes(etb, context=1, tb_offset=0):
135135
aux = traceback.extract_tb(etb)
136136
assert len(records) == len(aux)
137137
for i, (file, lnum, _, _) in enumerate(aux):
138-
maybeStart = lnum - 1 - context // 2
139-
start = max(maybeStart, 0)
138+
maybe_start = lnum - 1 - context // 2
139+
start = max(maybe_start, 0)
140140
end = start + context
141141
lines = linecache.getlines(file)[start:end]
142-
# pad with empty lines if necessary
143-
if maybeStart < 0:
144-
lines = (['\n'] * -maybeStart) + lines
145-
if len(lines) < context:
146-
lines += ['\n'] * (context - len(lines))
147142
buf = list(records[i])
148143
buf[LNUM_POS] = lnum
149144
buf[INDEX_POS] = lnum - 1 - start
@@ -355,13 +350,7 @@ def format_exc(etype, evalue, etb, context=5, tb_offset=0):
355350
pyver)
356351

357352
# Drop topmost frames if requested
358-
try:
359-
records = _fixed_getframes(etb, context, tb_offset)
360-
except:
361-
raise
362-
print('\nUnfortunately, your original traceback can not be '
363-
'constructed.\n')
364-
return ''
353+
records = _fixed_getframes(etb, context, tb_offset)
365354

366355
# Get (safely) a string form of the exception info
367356
try:
@@ -397,18 +386,13 @@ def format_outer_frames(context=5, stack_start=None, stack_end=None,
397386
filename = filename[:-4] + '.py'
398387
if ignore_ipython:
399388
# Hack to avoid printing the internals of IPython
400-
if (os.path.basename(filename) == 'iplib.py'
401-
and func_name in ('safe_execfile', 'runcode')):
389+
if (os.path.basename(filename) in ('iplib.py', 'py3compat.py')
390+
and func_name in ('execfile', 'safe_execfile', 'runcode')):
402391
break
403-
maybeStart = line_no - 1 - context // 2
404-
start = max(maybeStart, 0)
392+
maybe_start = line_no - 1 - context // 2
393+
start = max(maybe_start, 0)
405394
end = start + context
406395
lines = linecache.getlines(filename)[start:end]
407-
# pad with empty lines if necessary
408-
if maybeStart < 0:
409-
lines = (['\n'] * -maybeStart) + lines
410-
if len(lines) < context:
411-
lines += ['\n'] * (context - len(lines))
412396
buf = list(records[i])
413397
buf[LNUM_POS] = line_no
414398
buf[INDEX_POS] = line_no - 1 - start

‎sklearn/externals/joblib/func_inspect.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ def _signature_str(function_name, arg_spec):
190190
arg_spec_for_format = arg_spec[:7 if PY3_OR_LATER else 4]
191191

192192
arg_spec_str = inspect.formatargspec(*arg_spec_for_format)
193-
return '{0}{1}'.format(function_name, arg_spec_str)
193+
return '{}{}'.format(function_name, arg_spec_str)
194194

195195

196196
def _function_called_str(function_name, args, kwargs):
@@ -316,6 +316,13 @@ def filter_args(func, ignore_lst, args=(), kwargs=dict()):
316316
return arg_dict
317317

318318

319+
def _format_arg(arg):
320+
formatted_arg = pformat(arg, indent=2)
321+
if len(formatted_arg) > 1500:
322+
formatted_arg = '%s...' % formatted_arg[:700]
323+
return formatted_arg
324+
325+
319326
def format_signature(func, *args, **kwargs):
320327
# XXX: Should this use inspect.formatargvalues/formatargspec?
321328
module, name = get_func_name(func)
@@ -328,14 +335,12 @@ def format_signature(func, *args, **kwargs):
328335
arg_str = list()
329336
previous_length = 0
330337
for arg in args:
331-
arg = pformat(arg, indent=2)
332-
if len(arg) > 1500:
333-
arg = '%s...' % arg[:700]
338+
formatted_arg = _format_arg(arg)
334339
if previous_length > 80:
335-
arg = '\n%s' % arg
336-
previous_length = len(arg)
337-
arg_str.append(arg)
338-
arg_str.extend(['%s=%s' % (v, pformat(i)) for v, i in kwargs.items()])
340+
formatted_arg = '\n%s' % formatted_arg
341+
previous_length = len(formatted_arg)
342+
arg_str.append(formatted_arg)
343+
arg_str.extend(['%s=%s' % (v, _format_arg(i)) for v, i in kwargs.items()])
339344
arg_str = ', '.join(arg_str)
340345

341346
signature = '%s(%s)' % (name, arg_str)

‎sklearn/externals/joblib/hashing.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import types
1414
import struct
1515
import io
16+
import decimal
1617

1718
from ._compat import _bytes_or_unicode, PY3_OR_LATER
1819

@@ -35,7 +36,7 @@ def __init__(self, set_sequence):
3536
# This fails on python 3 when elements are unorderable
3637
# but we keep it in a try as it's faster.
3738
self._sequence = sorted(set_sequence)
38-
except TypeError:
39+
except (TypeError, decimal.InvalidOperation):
3940
# If elements are unorderable, sorting them using their hash.
4041
# This is slower but works in any case.
4142
self._sequence = sorted((hash(e) for e in set_sequence))

‎sklearn/externals/joblib/logger.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def __init__(self, depth=3):
7474
self.depth = depth
7575

7676
def warn(self, msg):
77-
logging.warn("[%s]: %s" % (self, msg))
77+
logging.warning("[%s]: %s" % (self, msg))
7878

7979
def debug(self, msg):
8080
# XXX: This conflicts with the debug flag used in children class

0 commit comments

Comments
 (0)