MNT Bump joblib version dependency to 1.0.0 (#22365)

thomasjpfan · web-flow · commit 5f75acdd12d7 · 2022-02-09T13:47:18.000+01:00
diff --git a/README.rst b/README.rst
@@ -35,7 +35,7 @@
 .. |PythonMinVersion| replace:: 3.7
 .. |NumPyMinVersion| replace:: 1.14.6
 .. |SciPyMinVersion| replace:: 1.1.0
-.. |JoblibMinVersion| replace:: 0.11
+.. |JoblibMinVersion| replace:: 1.0.0
 .. |ThreadpoolctlMinVersion| replace:: 2.0.0
 .. |MatplotlibMinVersion| replace:: 2.2.3
 .. |Scikit-ImageMinVersion| replace:: 0.14.5
diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py
@@ -64,11 +64,7 @@ def cythonize_extensions(top_path, config):
     with contextlib.suppress(ImportError):
         import joblib
 
-        if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"):
-            # earlier joblib versions don't account for CPU affinity
-            # constraints, and may over-estimate the number of available
-            # CPU particularly in CI (cf loky#114)
-            n_jobs = joblib.cpu_count()
+        n_jobs = joblib.cpu_count()
 
     # Additional checks for Cython
     cython_enable_debug_directives = (
diff --git a/sklearn/_min_dependencies.py b/sklearn/_min_dependencies.py
@@ -17,7 +17,7 @@
     NUMPY_MIN_VERSION = "1.14.6"
 
 SCIPY_MIN_VERSION = "1.1.0"
-JOBLIB_MIN_VERSION = "0.11"
+JOBLIB_MIN_VERSION = "1.0.0"
 THREADPOOLCTL_MIN_VERSION = "2.0.0"
 PYTEST_MIN_VERSION = "5.0.1"
 CYTHON_MIN_VERSION = "0.29.24"
diff --git a/sklearn/ensemble/_forest.py b/sklearn/ensemble/_forest.py
@@ -65,7 +65,6 @@ class calls the ``fit`` method of each sub-estimator on random samples
 from ..exceptions import DataConversionWarning
 from ._base import BaseEnsemble, _partition_estimators
 from ..utils.fixes import delayed
-from ..utils.fixes import _joblib_parallel_args
 from ..utils.multiclass import check_classification_targets, type_of_target
 from ..utils.validation import check_is_fitted, _check_sample_weight
 from ..utils.validation import _num_samples
@@ -249,7 +248,7 @@ def apply(self, X):
         results = Parallel(
             n_jobs=self.n_jobs,
             verbose=self.verbose,
-            **_joblib_parallel_args(prefer="threads"),
+            prefer="threads",
         )(delayed(tree.apply)(X, check_input=False) for tree in self.estimators_)
 
         return np.array(results).T
@@ -282,7 +281,7 @@ def decision_path(self, X):
         indicators = Parallel(
             n_jobs=self.n_jobs,
             verbose=self.verbose,
-            **_joblib_parallel_args(prefer="threads"),
+            prefer="threads",
         )(
             delayed(tree.decision_path)(X, check_input=False)
             for tree in self.estimators_
@@ -471,7 +470,7 @@ def fit(self, X, y, sample_weight=None):
             trees = Parallel(
                 n_jobs=self.n_jobs,
                 verbose=self.verbose,
-                **_joblib_parallel_args(prefer="threads"),
+                prefer="threads",
             )(
                 delayed(_parallel_build_trees)(
                     t,
@@ -625,9 +624,7 @@ def feature_importances_(self):
         """
         check_is_fitted(self)
 
-        all_importances = Parallel(
-            n_jobs=self.n_jobs, **_joblib_parallel_args(prefer="threads")
-        )(
+        all_importances = Parallel(n_jobs=self.n_jobs, prefer="threads")(
             delayed(getattr)(tree, "feature_importances_")
             for tree in self.estimators_
             if tree.tree_.node_count > 1
@@ -879,11 +876,7 @@ def predict_proba(self, X):
             for j in np.atleast_1d(self.n_classes_)
         ]
         lock = threading.Lock()
-        Parallel(
-            n_jobs=n_jobs,
-            verbose=self.verbose,
-            **_joblib_parallel_args(require="sharedmem"),
-        )(
+        Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")(
             delayed(_accumulate_prediction)(e.predict_proba, X, all_proba, lock)
             for e in self.estimators_
         )
@@ -1002,11 +995,7 @@ def predict(self, X):
 
         # Parallel loop
         lock = threading.Lock()
-        Parallel(
-            n_jobs=n_jobs,
-            verbose=self.verbose,
-            **_joblib_parallel_args(require="sharedmem"),
-        )(
+        Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")(
             delayed(_accumulate_prediction)(e.predict, X, [y_hat], lock)
             for e in self.estimators_
         )
diff --git a/sklearn/ensemble/_iforest.py b/sklearn/ensemble/_iforest.py
@@ -14,7 +14,6 @@
     gen_batches,
     get_chunk_n_rows,
 )
-from ..utils.fixes import _joblib_parallel_args
 from ..utils.validation import check_is_fitted, _num_samples
 from ..base import OutlierMixin
 
@@ -231,7 +230,7 @@ def _parallel_args(self):
         # a thread-based backend rather than a process-based backend so as
         # to avoid suffering from communication overhead and extra memory
         # copies.
-        return _joblib_parallel_args(prefer="threads")
+        return {"prefer": "threads"}
 
     def fit(self, X, y=None, sample_weight=None):
         """
diff --git a/sklearn/linear_model/_coordinate_descent.py b/sklearn/linear_model/_coordinate_descent.py
@@ -22,7 +22,7 @@
 from ..utils.validation import check_random_state
 from ..model_selection import check_cv
 from ..utils.extmath import safe_sparse_dot
-from ..utils.fixes import _astype_copy_false, _joblib_parallel_args
+from ..utils.fixes import _astype_copy_false
 from ..utils.validation import (
     _check_sample_weight,
     check_consistent_length,
@@ -1692,7 +1692,7 @@ def fit(self, X, y, sample_weight=None):
         mse_paths = Parallel(
             n_jobs=self.n_jobs,
             verbose=self.verbose,
-            **_joblib_parallel_args(prefer="threads"),
+            prefer="threads",
         )(jobs)
         mse_paths = np.reshape(mse_paths, (n_l1_ratio, len(folds), -1))
         # The mean is computed over folds.
diff --git a/sklearn/linear_model/_logistic.py b/sklearn/linear_model/_logistic.py
@@ -29,7 +29,6 @@
 from ..utils.optimize import _newton_cg, _check_optimize_result
 from ..utils.validation import check_is_fitted, _check_sample_weight
 from ..utils.multiclass import check_classification_targets
-from ..utils.fixes import _joblib_parallel_args
 from ..utils.fixes import delayed
 from ..model_selection import check_cv
 from ..metrics import get_scorer
@@ -1585,11 +1584,7 @@ def fit(self, X, y, sample_weight=None):
             prefer = "threads"
         else:
             prefer = "processes"
-        fold_coefs_ = Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            **_joblib_parallel_args(prefer=prefer),
-        )(
+        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, prefer=prefer)(
             path_func(
                 X,
                 y,
@@ -2150,11 +2145,7 @@ def fit(self, X, y, sample_weight=None):
         else:
             prefer = "processes"
 
-        fold_coefs_ = Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            **_joblib_parallel_args(prefer=prefer),
-        )(
+        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, prefer=prefer)(
             path_func(
                 X,
                 y,
diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py
@@ -36,7 +36,6 @@
 from ._sgd_fast import Huber
 from ._sgd_fast import EpsilonInsensitive
 from ._sgd_fast import SquaredEpsilonInsensitive
-from ..utils.fixes import _joblib_parallel_args
 
 LEARNING_RATE_TYPES = {
     "constant": 1,
@@ -752,9 +751,7 @@ def _fit_multiclass(self, X, y, alpha, C, learning_rate, sample_weight, max_iter
         random_state = check_random_state(self.random_state)
         seeds = random_state.randint(MAX_INT, size=len(self.classes_))
         result = Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            **_joblib_parallel_args(require="sharedmem"),
+            n_jobs=self.n_jobs, verbose=self.verbose, require="sharedmem"
         )(
             delayed(fit_binary)(
                 self,
diff --git a/sklearn/neighbors/_base.py b/sklearn/neighbors/_base.py
@@ -14,7 +14,6 @@
 
 import numpy as np
 from scipy.sparse import csr_matrix, issparse
-import joblib
 from joblib import Parallel, effective_n_jobs
 
 from ._ball_tree import BallTree
@@ -795,13 +794,7 @@ class from an array representing our data set and ask who's
                     "or set algorithm='brute'"
                     % self._fit_method
                 )
-            old_joblib = parse_version(joblib.__version__) < parse_version("0.12")
-            if old_joblib:
-                # Deal with change of API in joblib
-                parallel_kwargs = {"backend": "threading"}
-            else:
-                parallel_kwargs = {"prefer": "threads"}
-            chunked_results = Parallel(n_jobs, **parallel_kwargs)(
+            chunked_results = Parallel(n_jobs, prefer="threads")(
                 delayed(_tree_query_parallel_helper)(
                     self._tree, X[s], n_neighbors, return_distance
                 )
@@ -1133,13 +1126,7 @@ class from an array representing our data set and ask who's
 
             n_jobs = effective_n_jobs(self.n_jobs)
             delayed_query = delayed(_tree_query_radius_parallel_helper)
-            if parse_version(joblib.__version__) < parse_version("0.12"):
-                # Deal with change of API in joblib
-                parallel_kwargs = {"backend": "threading"}
-            else:
-                parallel_kwargs = {"prefer": "threads"}
-
-            chunked_results = Parallel(n_jobs, **parallel_kwargs)(
+            chunked_results = Parallel(n_jobs, prefer="threads")(
                 delayed_query(
                     self._tree, X[s], radius, return_distance, sort_results=sort_results
                 )
diff --git a/sklearn/pipeline.py b/sklearn/pipeline.py
@@ -326,22 +326,10 @@ def _fit(self, X, y=None, **fit_params_steps):
                 with _print_elapsed_time("Pipeline", self._log_message(step_idx)):
                     continue
 
-            if hasattr(memory, "location"):
-                # joblib >= 0.12
-                if memory.location is None:
-                    # we do not clone when caching is disabled to
-                    # preserve backward compatibility
-                    cloned_transformer = transformer
-                else:
-                    cloned_transformer = clone(transformer)
-            elif hasattr(memory, "cachedir"):
-                # joblib < 0.11
-                if memory.cachedir is None:
-                    # we do not clone when caching is disabled to
-                    # preserve backward compatibility
-                    cloned_transformer = transformer
-                else:
-                    cloned_transformer = clone(transformer)
+            if hasattr(memory, "location") and memory.location is None:
+                # we do not clone when caching is disabled to
+                # preserve backward compatibility
+                cloned_transformer = transformer
             else:
                 cloned_transformer = clone(transformer)
             # Fit or load from cache the current transformer
diff --git a/sklearn/tests/test_config.py b/sklearn/tests/test_config.py
@@ -2,12 +2,10 @@
 from concurrent.futures import ThreadPoolExecutor
 
 from joblib import Parallel
-import joblib
 import pytest
 
 from sklearn import get_config, set_config, config_context
 from sklearn.utils.fixes import delayed
-from sklearn.utils.fixes import parse_version
 
 
 def test_config_context():
@@ -110,10 +108,6 @@ def test_config_threadsafe_joblib(backend):
     should be the same as the value passed to the function. In other words,
     it is not influenced by the other job setting assume_finite to True.
     """
-
-    if parse_version(joblib.__version__) < parse_version("0.12") and backend == "loky":
-        pytest.skip("loky backend does not exist in joblib <0.12")  # noqa
-
     assume_finites = [False, True]
     sleep_durations = [0.1, 0.2]
 
diff --git a/sklearn/tests/test_pipeline.py b/sklearn/tests/test_pipeline.py
@@ -12,7 +12,6 @@
 from scipy import sparse
 import joblib
 
-from sklearn.utils.fixes import parse_version
 from sklearn.utils._testing import (
     assert_allclose,
     assert_array_equal,
@@ -1163,11 +1162,7 @@ def test_pipeline_memory():
     y = iris.target
     cachedir = mkdtemp()
     try:
-        if parse_version(joblib.__version__) < parse_version("0.12"):
-            # Deal with change of API in joblib
-            memory = joblib.Memory(cachedir=cachedir, verbose=10)
-        else:
-            memory = joblib.Memory(location=cachedir, verbose=10)
+        memory = joblib.Memory(location=cachedir, verbose=10)
         # Test with Transformer + SVC
         clf = SVC(probability=True, random_state=0)
         transf = DummyTransf()
@@ -1227,11 +1222,7 @@ def test_pipeline_memory():
 
 def test_make_pipeline_memory():
     cachedir = mkdtemp()
-    if parse_version(joblib.__version__) < parse_version("0.12"):
-        # Deal with change of API in joblib
-        memory = joblib.Memory(cachedir=cachedir, verbose=10)
-    else:
-        memory = joblib.Memory(location=cachedir, verbose=10)
+    memory = joblib.Memory(location=cachedir, verbose=10)
     pipeline = make_pipeline(DummyTransf(), SVC(), memory=memory)
     assert pipeline.memory is memory
     pipeline = make_pipeline(DummyTransf(), SVC())
diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py
@@ -58,58 +58,6 @@ def _astype_copy_false(X):
         return {}
 
 
-def _joblib_parallel_args(**kwargs):
-    """Set joblib.Parallel arguments in a compatible way for 0.11 and 0.12+
-
-    For joblib 0.11 this maps both ``prefer`` and ``require`` parameters to
-    a specific ``backend``.
-
-    Parameters
-    ----------
-
-    prefer : str in {'processes', 'threads'} or None
-        Soft hint to choose the default backend if no specific backend
-        was selected with the parallel_backend context manager.
-
-    require : 'sharedmem' or None
-        Hard condstraint to select the backend. If set to 'sharedmem',
-        the selected backend will be single-host and thread-based even
-        if the user asked for a non-thread based backend with
-        parallel_backend.
-
-    See joblib.Parallel documentation for more details
-    """
-    import joblib
-
-    if parse_version(joblib.__version__) >= parse_version("0.12"):
-        return kwargs
-
-    extra_args = set(kwargs.keys()).difference({"prefer", "require"})
-    if extra_args:
-        raise NotImplementedError(
-            "unhandled arguments %s with joblib %s"
-            % (list(extra_args), joblib.__version__)
-        )
-    args = {}
-    if "prefer" in kwargs:
-        prefer = kwargs["prefer"]
-        if prefer not in ["threads", "processes", None]:
-            raise ValueError("prefer=%s is not supported" % prefer)
-        args["backend"] = {
-            "threads": "threading",
-            "processes": "multiprocessing",
-            None: None,
-        }[prefer]
-
-    if "require" in kwargs:
-        require = kwargs["require"]
-        if require not in [None, "sharedmem"]:
-            raise ValueError("require=%s is not supported" % require)
-        if require == "sharedmem":
-            args["backend"] = "threading"
-    return args
-
-
 class loguniform(scipy.stats.reciprocal):
     """A class supporting log-uniform random variables.
 
diff --git a/sklearn/utils/tests/test_fixes.py b/sklearn/utils/tests/test_fixes.py
diff --git a/sklearn/utils/tests/test_parallel.py b/sklearn/utils/tests/test_parallel.py
diff --git a/sklearn/utils/tests/test_validation.py b/sklearn/utils/tests/test_validation.py
diff --git a/sklearn/utils/validation.py b/sklearn/utils/validation.py
