From a0d0b53a476e7dd5e50fa7a392029e27d14396a4 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 29 Sep 2021 14:34:03 +0200 Subject: [PATCH 1/9] wip --- sklearn/utils/fixes.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index ee84f632abaae..043d93b0afd44 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -18,6 +18,7 @@ import scipy import scipy.stats from scipy.sparse.linalg import lsqr as sparse_lsqr # noqa +import threadpoolctl from .._config import config_context, get_config from ..externals._packaging.version import parse as parse_version @@ -271,3 +272,12 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis dtype=dtype, axis=axis, ) + + +if parse_version(threadpoolctl.__version__) >= parse_version("3.0.0"): + from threadpoolctl import ThreadpoolController + print("once") + _sklearn_threadpool_controller = ThreadpoolController() +else: + print("twice") + _sklearn_threadpool_controller = None From 9a6e2d24dcfa1937e72e3edfe35729391bb71fdd Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 29 Sep 2021 17:06:22 +0200 Subject: [PATCH 2/9] wip --- sklearn/utils/_threadpoolctl_fixes.py | 15 +++++++++++++++ sklearn/utils/fixes.py | 23 ++++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 sklearn/utils/_threadpoolctl_fixes.py diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py new file mode 100644 index 0000000000000..94269497f73df --- /dev/null +++ b/sklearn/utils/_threadpoolctl_fixes.py @@ -0,0 +1,15 @@ +"""Compatibility fixes for older version of threadpoolctl + +This module is separated from utils.fixes because it need to not be imported too soon +when importing sklearn but utils.fixes is. +""" + +import threadpoolctl + + +if hasattr(threadpoolctl, "ThreadpoolController"): + _sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() + threadpool_limits = _sklearn_threadpool_controller.limit +else: + _sklearn_threadpool_controller = None + threadpool_limits = threadpoolctl.threadpool_limits diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 043d93b0afd44..4f53100e9a270 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -13,6 +13,8 @@ from functools import update_wrapper import functools +import sklearn + import numpy as np import scipy.sparse as sp import scipy @@ -273,11 +275,18 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis axis=axis, ) + +# if hasattr(threadpoolctl, "ThreadpoolController"): +# _sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() +# else: +# _sklearn_threadpool_controller = None -if parse_version(threadpoolctl.__version__) >= parse_version("3.0.0"): - from threadpoolctl import ThreadpoolController - print("once") - _sklearn_threadpool_controller = ThreadpoolController() -else: - print("twice") - _sklearn_threadpool_controller = None + +# def threadpool_limits(limits=None, user_api=None): +# from ..import _sklearn_threadpool_controller +# if _sklearn_threadpool_controller is None: +# return threadpoolctl.threadpool_limits(limits=limits, user_api=user_api) +# else: +# return _sklearn_threadpool_controller.limit(limits=limits, user_api=user_api) + +# if hasattr(threadpoolctl, "ThreadpoolController"): From 87c3fef8db8744018a3837ffc24f31406623357d Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 30 Sep 2021 17:14:09 +0200 Subject: [PATCH 3/9] helper to use the new threadpoolctl API when possible --- sklearn/utils/_threadpoolctl_fixes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py index 94269497f73df..90b7d4839f1ea 100644 --- a/sklearn/utils/_threadpoolctl_fixes.py +++ b/sklearn/utils/_threadpoolctl_fixes.py @@ -10,6 +10,8 @@ if hasattr(threadpoolctl, "ThreadpoolController"): _sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() threadpool_limits = _sklearn_threadpool_controller.limit + threadpool_info = _sklearn_threadpool_controller.info else: _sklearn_threadpool_controller = None threadpool_limits = threadpoolctl.threadpool_limits + threadpool_info = threadpoolctl.threadpool_info From b571590ae67e397963856660d20fe8bd5b6f560e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 30 Sep 2021 17:19:38 +0200 Subject: [PATCH 4/9] cln --- sklearn/utils/_threadpoolctl_fixes.py | 2 ++ sklearn/utils/fixes.py | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py index 90b7d4839f1ea..b91125e518802 100644 --- a/sklearn/utils/_threadpoolctl_fixes.py +++ b/sklearn/utils/_threadpoolctl_fixes.py @@ -4,6 +4,8 @@ when importing sklearn but utils.fixes is. """ +"""Compatibility fixes for older version of threadpoolctl""" + import threadpoolctl diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 4f53100e9a270..53b79ffbde16e 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -20,7 +20,6 @@ import scipy import scipy.stats from scipy.sparse.linalg import lsqr as sparse_lsqr # noqa -import threadpoolctl from .._config import config_context, get_config from ..externals._packaging.version import parse as parse_version @@ -274,19 +273,3 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis dtype=dtype, axis=axis, ) - - -# if hasattr(threadpoolctl, "ThreadpoolController"): -# _sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() -# else: -# _sklearn_threadpool_controller = None - - -# def threadpool_limits(limits=None, user_api=None): -# from ..import _sklearn_threadpool_controller -# if _sklearn_threadpool_controller is None: -# return threadpoolctl.threadpool_limits(limits=limits, user_api=user_api) -# else: -# return _sklearn_threadpool_controller.limit(limits=limits, user_api=user_api) - -# if hasattr(threadpoolctl, "ThreadpoolController"): From 6cc66f2443e3d99e24219b6515df3a706402c94a Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 30 Sep 2021 17:20:10 +0200 Subject: [PATCH 5/9] cln --- sklearn/utils/fixes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 53b79ffbde16e..ee84f632abaae 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -13,8 +13,6 @@ from functools import update_wrapper import functools -import sklearn - import numpy as np import scipy.sparse as sp import scipy From 69ccc30af51ddd903da16883ca969c0c4f796abe Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 30 Sep 2021 17:35:33 +0200 Subject: [PATCH 6/9] cln --- sklearn/utils/_threadpoolctl_fixes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py index b91125e518802..90b7d4839f1ea 100644 --- a/sklearn/utils/_threadpoolctl_fixes.py +++ b/sklearn/utils/_threadpoolctl_fixes.py @@ -4,8 +4,6 @@ when importing sklearn but utils.fixes is. """ -"""Compatibility fixes for older version of threadpoolctl""" - import threadpoolctl From 26ab63da71b689de9cd287a67c031d5557a002b7 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 30 Sep 2021 17:37:11 +0200 Subject: [PATCH 7/9] cln --- sklearn/utils/_threadpoolctl_fixes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py index 90b7d4839f1ea..9f49675d8487a 100644 --- a/sklearn/utils/_threadpoolctl_fixes.py +++ b/sklearn/utils/_threadpoolctl_fixes.py @@ -1,7 +1,7 @@ """Compatibility fixes for older version of threadpoolctl -This module is separated from utils.fixes because it need to not be imported too soon -when importing sklearn but utils.fixes is. +This module is separated from utils.fixes because it needs to not be imported too soon +when importing sklearn and it happens that utils.fixes is. """ import threadpoolctl From e0a7b1fa95dabb4420e39d44924b4ebf7dd8f3ff Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 1 Oct 2021 13:28:33 +0200 Subject: [PATCH 8/9] change logic, lazy creation of the controller --- sklearn/utils/_threadpoolctl_fixes.py | 17 -------------- sklearn/utils/fixes.py | 32 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 17 deletions(-) delete mode 100644 sklearn/utils/_threadpoolctl_fixes.py diff --git a/sklearn/utils/_threadpoolctl_fixes.py b/sklearn/utils/_threadpoolctl_fixes.py deleted file mode 100644 index 9f49675d8487a..0000000000000 --- a/sklearn/utils/_threadpoolctl_fixes.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Compatibility fixes for older version of threadpoolctl - -This module is separated from utils.fixes because it needs to not be imported too soon -when importing sklearn and it happens that utils.fixes is. -""" - -import threadpoolctl - - -if hasattr(threadpoolctl, "ThreadpoolController"): - _sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() - threadpool_limits = _sklearn_threadpool_controller.limit - threadpool_info = _sklearn_threadpool_controller.info -else: - _sklearn_threadpool_controller = None - threadpool_limits = threadpoolctl.threadpool_limits - threadpool_info = threadpoolctl.threadpool_info diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index ee84f632abaae..feb9845fd7818 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -13,11 +13,13 @@ from functools import update_wrapper import functools +import sklearn import numpy as np import scipy.sparse as sp import scipy import scipy.stats from scipy.sparse.linalg import lsqr as sparse_lsqr # noqa +import threadpoolctl from .._config import config_context, get_config from ..externals._packaging.version import parse as parse_version @@ -271,3 +273,33 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis dtype=dtype, axis=axis, ) + + +# compatibility fix for threadpoolctl >= 3.0.0 +# since version 3 it's possible to setup a global threadpool controller to avoid +# looping through all loaded shared libraries each time. +# the global controller is created during the first call to threadpoolctl. +def _get_threadpool_controller(): + if not hasattr(threadpoolctl, "ThreadpoolController"): + return None + + if not hasattr(sklearn, "_sklearn_threadpool_controller"): + sklearn._sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() + + return sklearn._sklearn_threadpool_controller + + +def threadpool_limits(limits=None, user_api=None): + controller = _get_threadpool_controller() + if controller is not None: + return controller.limit(limits=limits, user_api=user_api) + else: + return threadpoolctl.threadpool_limits(limits=limits, user_api=user_api) + + +def threadpool_info(): + controller = _get_threadpool_controller() + if controller is not None: + return controller.info() + else: + return threadpoolctl.threadpool_info() From 3b8fc795e9ba64bcf2dbac77924e54ccc3d0e78c Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Mon, 4 Oct 2021 17:01:36 +0200 Subject: [PATCH 9/9] use it the code base --- sklearn/cluster/_kmeans.py | 4 ++-- sklearn/cluster/tests/test_k_means.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/cluster/_kmeans.py b/sklearn/cluster/_kmeans.py index 6ccc6e11cb7de..1d8d889696e4e 100644 --- a/sklearn/cluster/_kmeans.py +++ b/sklearn/cluster/_kmeans.py @@ -15,13 +15,13 @@ import numpy as np import scipy.sparse as sp -from threadpoolctl import threadpool_limits -from threadpoolctl import threadpool_info from ..base import BaseEstimator, ClusterMixin, TransformerMixin from ..metrics.pairwise import euclidean_distances from ..metrics.pairwise import _euclidean_distances from ..utils.extmath import row_norms, stable_cumsum +from ..utils.fixes import threadpool_limits +from ..utils.fixes import threadpool_info from ..utils.sparsefuncs_fast import assign_rows_csr from ..utils.sparsefuncs import mean_variance_axis from ..utils import check_array diff --git a/sklearn/cluster/tests/test_k_means.py b/sklearn/cluster/tests/test_k_means.py index cbac3974c1e11..d02f67333acca 100644 --- a/sklearn/cluster/tests/test_k_means.py +++ b/sklearn/cluster/tests/test_k_means.py @@ -4,13 +4,13 @@ import numpy as np from scipy import sparse as sp -from threadpoolctl import threadpool_limits import pytest from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_allclose from sklearn.utils.fixes import _astype_copy_false +from sklearn.utils.fixes import threadpool_limits from sklearn.base import clone from sklearn.exceptions import ConvergenceWarning