diff --git a/README.rst b/README.rst index 3f9a4ad726806..4ac297063c26e 100644 --- a/README.rst +++ b/README.rst @@ -36,7 +36,7 @@ .. |NumPyMinVersion| replace:: 1.19.5 .. |SciPyMinVersion| replace:: 1.6.0 .. |JoblibMinVersion| replace:: 1.2.0 -.. |ThreadpoolctlMinVersion| replace:: 2.0.0 +.. |ThreadpoolctlMinVersion| replace:: 3.1.0 .. |MatplotlibMinVersion| replace:: 3.3.4 .. |Scikit-ImageMinVersion| replace:: 0.17.2 .. |PandasMinVersion| replace:: 1.1.5 diff --git a/build_tools/azure/debian_atlas_32bit_lock.txt b/build_tools/azure/debian_atlas_32bit_lock.txt index a6bb0485d1933..5bff365bb1eea 100644 --- a/build_tools/azure/debian_atlas_32bit_lock.txt +++ b/build_tools/azure/debian_atlas_32bit_lock.txt @@ -37,7 +37,7 @@ pytest==7.1.2 # pytest-cov pytest-cov==2.9.0 # via -r build_tools/azure/debian_atlas_32bit_requirements.txt -threadpoolctl==2.2.0 +threadpoolctl==3.1.0 # via -r build_tools/azure/debian_atlas_32bit_requirements.txt tomli==2.0.1 # via diff --git a/build_tools/azure/debian_atlas_32bit_requirements.txt b/build_tools/azure/debian_atlas_32bit_requirements.txt index d1bc22529d4f4..615193a71fc6b 100644 --- a/build_tools/azure/debian_atlas_32bit_requirements.txt +++ b/build_tools/azure/debian_atlas_32bit_requirements.txt @@ -3,7 +3,7 @@ # build_tools/update_environments_and_lock_files.py cython==3.0.10 # min joblib==1.2.0 # min -threadpoolctl==2.2.0 +threadpoolctl==3.1.0 pytest==7.1.2 # min pytest-cov==2.9.0 # min ninja diff --git a/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml index 9c46400c2d3c6..01bd378aa121a 100644 --- a/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml +++ b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml @@ -9,7 +9,6 @@ dependencies: - blas[build=mkl] - scipy<1.12 - joblib - - threadpoolctl - matplotlib - pandas - pyamg @@ -25,3 +24,4 @@ dependencies: - pip - pip: - cython + - threadpoolctl diff --git a/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock b/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock index a9c9b364426a7..7da0e513ac143 100644 --- a/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock +++ b/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: osx-64 -# input_hash: d3fadab6d5d5d715660beb53286e2687b018b5195ef7ce47928bb79a96ee851b +# input_hash: e0d2cf2593df1f2c6969d68cf849136bee785b51f6cfc50ea1bdca2143d4a051 @EXPLICIT https://repo.anaconda.com/pkgs/main/osx-64/blas-1.0-mkl.conda#cb2c87e85ac8e0ceae776d26d4214c8a https://repo.anaconda.com/pkgs/main/osx-64/bzip2-1.0.8-h6c40b1e_5.conda#0f51dde96c82dcf58a788787fed4c5b9 @@ -56,7 +56,6 @@ https://repo.anaconda.com/pkgs/main/noarch/python-tzdata-2023.3-pyhd3eb1b0_0.con https://repo.anaconda.com/pkgs/main/osx-64/pytz-2023.3.post1-py312hecd8cb5_0.conda#2636382c9a424f69cbc36b1c5dc1f2fc https://repo.anaconda.com/pkgs/main/osx-64/setuptools-68.2.2-py312hecd8cb5_0.conda#64235f0c451427d86808c70c1c31cb8b https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_1.conda#34586824d411d36af2fa40e799c172d0 -https://repo.anaconda.com/pkgs/main/noarch/threadpoolctl-2.2.0-pyh0d69192_0.conda#bbfdbae4934150b902f97daaf287efe2 https://repo.anaconda.com/pkgs/main/noarch/toml-0.10.2-pyhd3eb1b0_0.conda#cda05f5f6d8509529d1a2743288d197a https://repo.anaconda.com/pkgs/main/osx-64/tornado-6.3.3-py312h6c40b1e_0.conda#49173b5a36c9134865221f29d4a73fb6 https://repo.anaconda.com/pkgs/main/osx-64/unicodedata2-15.1.0-py312h6c40b1e_0.conda#65bd2cb787fc99662d9bb6e6520c5826 @@ -84,3 +83,4 @@ https://repo.anaconda.com/pkgs/main/osx-64/scipy-1.11.4-py312h81688c2_0.conda#7d https://repo.anaconda.com/pkgs/main/osx-64/pandas-2.2.1-py312he282a81_0.conda#021b70a1e40efb75b89eb8ebdb347132 https://repo.anaconda.com/pkgs/main/osx-64/pyamg-4.2.3-py312h44cbcf4_0.conda#3bdc7be74087b3a5a83c520a74e1e8eb # pip cython @ https://files.pythonhosted.org/packages/d5/6d/06c08d75adb98cdf72af18801e193d22580cc86ca553610f430f18ea26b3/Cython-3.0.10-cp312-cp312-macosx_10_9_x86_64.whl#sha256=8f2864ab5fcd27a346f0b50f901ebeb8f60b25a60a575ccfd982e7f3e9674914 +# pip threadpoolctl @ https://files.pythonhosted.org/packages/1e/84/ccd9b08653022b7785b6e3ee070ffb2825841e0dc119be22f0840b2b35cb/threadpoolctl-3.4.0-py3-none-any.whl#sha256=8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262 diff --git a/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock b/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock index b5f1d7613fa72..a4d18dc37716a 100644 --- a/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock +++ b/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock @@ -11,7 +11,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-11.2.0-h1234567_1.cond https://repo.anaconda.com/pkgs/main/linux-64/_openmp_mutex-5.1-1_gnu.conda#71d281e9c2192cb3fa425655a8defb85 https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-11.2.0-h1234567_1.conda#a87728dabf3151fb9cfa990bd2eb0464 https://repo.anaconda.com/pkgs/main/linux-64/bzip2-1.0.8-h5eee18b_5.conda#9c8dec113089c4aca7392c6a3864f505 -https://repo.anaconda.com/pkgs/main/linux-64/expat-2.5.0-h6a678d5_0.conda#9a21d99d49a0a556cf9590430dec8ec0 +https://repo.anaconda.com/pkgs/main/linux-64/expat-2.6.2-h6a678d5_0.conda#55049db2772dae035f6b8a95f72b5970 https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.4.4-h6a678d5_0.conda#06e288f9250abef59b9a367d151fc339 https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.41.5-h5eee18b_0.conda#4a6a2354414c9080327274aa514e5299 https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.4-h6a678d5_0.conda#5558eec6e2191741a92f832ea826251c @@ -22,7 +22,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/ccache-3.7.9-hfe4627d_0.conda#bef6f https://repo.anaconda.com/pkgs/main/linux-64/readline-8.2-h5eee18b_0.conda#be42180685cce6e6b0329201d9f48efb https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.12-h1ccaba5_0.conda#fa10ff4aa631fa4aa090a6234d7770b9 https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.41.2-h5eee18b_0.conda#c7086c9ceb6cfe1c4c729a774a2d88a5 -https://repo.anaconda.com/pkgs/main/linux-64/python-3.12.2-h996f2a0_0.conda#bc4748d0d26253c8499a3abeca289469 +https://repo.anaconda.com/pkgs/main/linux-64/python-3.12.3-h996f2a0_0.conda#77af2bd351a8311d1e780bcfa7819bb8 https://repo.anaconda.com/pkgs/main/linux-64/setuptools-68.2.2-py312h06a4308_0.conda#83ba634cde4f30d9e0b88e4ac9716ca4 https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.41.2-py312h06a4308_0.conda#b2c4f82880d58d679f3982370d80c0e2 https://repo.anaconda.com/pkgs/main/linux-64/pip-23.3.1-py312h06a4308_0.conda#e1d44bca4a257e84af33503233491107 @@ -31,7 +31,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/pip-23.3.1-py312h06a4308_0.conda#e1 # pip certifi @ https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl#sha256=dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1 # pip charset-normalizer @ https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b # pip coverage @ https://files.pythonhosted.org/packages/98/79/185cb42910b6a2b2851980407c8445ac0da0750dff65e420e86f973c8396/coverage-7.4.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51 -# pip docutils @ https://files.pythonhosted.org/packages/26/87/f238c0670b94533ac0353a4e2a1a771a0cc73277b88bff23d3ae35a256c1/docutils-0.20.1-py3-none-any.whl#sha256=96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6 +# pip docutils @ https://files.pythonhosted.org/packages/ea/6a/1c934b70bc12be40be886060709317c2ad5a5fe2180469410bd9344f5235/docutils-0.21.1-py3-none-any.whl#sha256=14c8d34a55b46c88f9f714adb29cefbdd69fb82f3fef825e59c5faab935390d8 # pip execnet @ https://files.pythonhosted.org/packages/43/09/2aea36ff60d16dd8879bdb2f5b3ee0ba8d08cbbdcdfe870e695ce3784385/execnet-2.1.1-py3-none-any.whl#sha256=26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc # pip idna @ https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl#sha256=82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # pip imagesize @ https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl#sha256=0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b @@ -41,7 +41,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/pip-23.3.1-py312h06a4308_0.conda#e1 # pip ninja @ https://files.pythonhosted.org/packages/6d/92/8d7aebd4430ab5ff65df2bfee6d5745f95c004284db2d8ca76dcbfd9de47/ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl#sha256=84502ec98f02a037a169c4b0d5d86075eaf6afc55e1879003d6cab51ced2ea4b # pip packaging @ https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl#sha256=2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 # pip platformdirs @ https://files.pythonhosted.org/packages/55/72/4898c44ee9ea6f43396fbc23d9bfaf3d06e01b83698bdf2e4c919deceb7c/platformdirs-4.2.0-py3-none-any.whl#sha256=0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 -# pip pluggy @ https://files.pythonhosted.org/packages/a5/5b/0cc789b59e8cc1bf288b38111d002d8c5917123194d45b29dcdac64723cc/pluggy-1.4.0-py3-none-any.whl#sha256=7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 +# pip pluggy @ https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl#sha256=44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 # pip pygments @ https://files.pythonhosted.org/packages/97/9c/372fef8377a6e340b1704768d20daaded98bf13282b5327beb2e2fe2c7ef/pygments-2.17.2-py3-none-any.whl#sha256=b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c # pip six @ https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl#sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # pip snowballstemmer @ https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl#sha256=c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a @@ -55,13 +55,13 @@ https://repo.anaconda.com/pkgs/main/linux-64/pip-23.3.1-py312h06a4308_0.conda#e1 # pip threadpoolctl @ https://files.pythonhosted.org/packages/1e/84/ccd9b08653022b7785b6e3ee070ffb2825841e0dc119be22f0840b2b35cb/threadpoolctl-3.4.0-py3-none-any.whl#sha256=8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262 # pip urllib3 @ https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.whl#sha256=450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d # pip jinja2 @ https://files.pythonhosted.org/packages/30/6d/6de6be2d02603ab56e72997708809e8a5b0fbfee080735109b40a3564843/Jinja2-3.1.3-py3-none-any.whl#sha256=7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa -# pip pyproject-metadata @ https://files.pythonhosted.org/packages/c4/cb/4678dfd70cd2f2d8969e571cdc1bb1e9293c698f8d1cf428fadcf48d6e9f/pyproject_metadata-0.7.1-py3-none-any.whl#sha256=28691fbb36266a819ec56c9fa1ecaf36f879d6944dfde5411e87fc4ff793aa60 +# pip pyproject-metadata @ https://files.pythonhosted.org/packages/aa/5f/bb5970d3d04173b46c9037109f7f05fc8904ff5be073ee49bb6ff00301bc/pyproject_metadata-0.8.0-py3-none-any.whl#sha256=ad858d448e1d3a1fb408ac5bac9ea7743e7a8bbb472f2693aaa334d2db42f526 # pip pytest @ https://files.pythonhosted.org/packages/51/ff/f6e8b8f39e08547faece4bd80f89d5a8de68a38b2d179cc1c4490ffa3286/pytest-7.4.4-py3-none-any.whl#sha256=b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8 # pip python-dateutil @ https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl#sha256=a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # pip requests @ https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl#sha256=58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f -# pip meson-python @ https://files.pythonhosted.org/packages/1f/60/b10b11ab470a690d5777310d6cfd1c9bdbbb0a1313a78c34a1e82e0b9d27/meson_python-0.15.0-py3-none-any.whl#sha256=3ae38253ff02b2e947a05e362a2eaf5a9a09d133c5666b4123399ee5fbf2e591 +# pip meson-python @ https://files.pythonhosted.org/packages/91/c0/104cb6244c83fe6bc3886f144cc433db0c0c78efac5dc00e409a5a08c87d/meson_python-0.16.0-py3-none-any.whl#sha256=842dc9f5dc29e55fc769ff1b6fe328412fe6c870220fc321060a1d2d395e69e8 # pip pooch @ https://files.pythonhosted.org/packages/f4/72/8ae0f1ba4ce6a4f6d4d01a60a9fdf690fde188c45c1872b0b4ddb0607ace/pooch-1.8.1-py3-none-any.whl#sha256=6b56611ac320c239faece1ac51a60b25796792599ce5c0b1bb87bf01df55e0a9 # pip pytest-cov @ https://files.pythonhosted.org/packages/78/3a/af5b4fa5961d9a1e6237b530eb87dd04aea6eb83da09d2a4073d81b54ccf/pytest_cov-5.0.0-py3-none-any.whl#sha256=4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652 -# pip pytest-xdist @ https://files.pythonhosted.org/packages/50/37/125fe5ec459321e2d48a0c38672cfc2419ad87d580196fd894e5f25230b0/pytest_xdist-3.5.0-py3-none-any.whl#sha256=d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24 -# pip sphinx @ https://files.pythonhosted.org/packages/b2/b6/8ed35256aa530a9d3da15d20bdc0ba888d5364441bb50a5a83ee7827affe/sphinx-7.2.6-py3-none-any.whl#sha256=1e09160a40b956dc623c910118fa636da93bd3ca0b9876a7b3df90f07d691560 +# pip pytest-xdist @ https://files.pythonhosted.org/packages/e0/df/585742c204227526baf641e45a95d3c0a94978b85a1414622c1017ebfcf8/pytest_xdist-3.6.0-py3-none-any.whl#sha256=958e08f38472e1b3a83450d8d3e682e90fdbffee39a97dd0f27185a3bd9074d1 +# pip sphinx @ https://files.pythonhosted.org/packages/b4/fa/130c32ed94cf270e3d0b9ded16fb7b2c8fea86fa7263c29a696a30c1dde7/sphinx-7.3.7-py3-none-any.whl#sha256=413f75440be4cacf328f580b4274ada4565fb2187d696a84970c23f77b64d8c3 # pip numpydoc @ https://files.pythonhosted.org/packages/f0/fa/dcfe0f65660661db757ee9ebd84e170ff98edd5d80235f62457d9088f85f/numpydoc-1.7.0-py3-none-any.whl#sha256=5a56419d931310d79a06cfc2a126d1558700feeb9b4f3d8dcae1a8134be829c9 diff --git a/build_tools/azure/pymin_conda_defaults_openblas_environment.yml b/build_tools/azure/pymin_conda_defaults_openblas_environment.yml index a422a0a539c53..17824c9b97074 100644 --- a/build_tools/azure/pymin_conda_defaults_openblas_environment.yml +++ b/build_tools/azure/pymin_conda_defaults_openblas_environment.yml @@ -9,7 +9,6 @@ dependencies: - blas[build=openblas] - scipy=1.7 - joblib - - threadpoolctl=2.2.0 - matplotlib=3.3.4 # min - pyamg - pytest<8 @@ -21,3 +20,4 @@ dependencies: - pip - pip: - cython==3.0.10 # min + - threadpoolctl diff --git a/build_tools/azure/pymin_conda_defaults_openblas_linux-64_conda.lock b/build_tools/azure/pymin_conda_defaults_openblas_linux-64_conda.lock index 00797394e4253..fef1871be2088 100644 --- a/build_tools/azure/pymin_conda_defaults_openblas_linux-64_conda.lock +++ b/build_tools/azure/pymin_conda_defaults_openblas_linux-64_conda.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 3d6bcb087065974114c1567c8dccd947a7376a7503b3514d82904299b651692d +# input_hash: 4f71ddcce93c9279161f5b016be417469aa5df726d06e4a1447c9270f60179e4 @EXPLICIT https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda#c3473ff8bdb3d124ed5ff11ec380d6f9 https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-openblas.conda#9ddfcaef10d79366c90128f5dc444be8 @@ -74,7 +74,6 @@ https://repo.anaconda.com/pkgs/main/linux-64/pyparsing-3.0.9-py39h06a4308_0.cond https://repo.anaconda.com/pkgs/main/linux-64/pyqt5-sip-12.13.0-py39h5eee18b_0.conda#256840c3841b52346ea5743be8490ede https://repo.anaconda.com/pkgs/main/linux-64/setuptools-68.2.2-py39h06a4308_0.conda#5b42cae5548732ae5c167bb1066085de https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_1.conda#34586824d411d36af2fa40e799c172d0 -https://repo.anaconda.com/pkgs/main/noarch/threadpoolctl-2.2.0-pyh0d69192_0.conda#bbfdbae4934150b902f97daaf287efe2 https://repo.anaconda.com/pkgs/main/noarch/toml-0.10.2-pyhd3eb1b0_0.conda#cda05f5f6d8509529d1a2743288d197a https://repo.anaconda.com/pkgs/main/linux-64/tomli-2.0.1-py39h06a4308_0.conda#b06dffe7ddca2645ed72f5116f0a087d https://repo.anaconda.com/pkgs/main/linux-64/tornado-6.3.3-py39h5eee18b_0.conda#9c4bd985bb8adcd12f47e790e95a9333 @@ -97,3 +96,4 @@ https://repo.anaconda.com/pkgs/main/linux-64/qt-main-5.15.2-h53bd1ea_10.conda#bd https://repo.anaconda.com/pkgs/main/linux-64/pyqt-5.15.10-py39h6a678d5_0.conda#52da5ff9b1144b078d2f41bab0b213f2 https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.3.4-py39h06a4308_0.conda#384fc5e01ebfcf30e7161119d3029b5a # pip cython @ https://files.pythonhosted.org/packages/a7/f5/3dde4d96076888ceaa981827b098274c2b45ddd4b20d75a8cfaa92b91eec/Cython-3.0.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=651a15a8534ebfb9b58cb0b87c269c70984b6f9c88bfe65e4f635f0e3f07dfcd +# pip threadpoolctl @ https://files.pythonhosted.org/packages/1e/84/ccd9b08653022b7785b6e3ee070ffb2825841e0dc119be22f0840b2b35cb/threadpoolctl-3.4.0-py3-none-any.whl#sha256=8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262 diff --git a/build_tools/azure/pypy3_linux-64_conda.lock b/build_tools/azure/pypy3_linux-64_conda.lock index 23710cfe35cb8..342bd1af08406 100644 --- a/build_tools/azure/pypy3_linux-64_conda.lock +++ b/build_tools/azure/pypy3_linux-64_conda.lock @@ -4,23 +4,23 @@ @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda#2f4327a1cbe7f022401b236e915a5fef -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_5.conda#f6f6600d18a4047b54f803cf708b868a +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda#3cfab3e709f77e9f1b3d380eb622494a https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.9-4_pypy39_pp73.conda#c1b2f29111681a4036ed21eaa3f44620 https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda#161081fc7cec0bfda0d86d7cb595f8d8 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_kmp_llvm.tar.bz2#562b26ba2e19059551a811e72ab7f793 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda#d4ff227c46917d3b4565302a2bbb276b +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda#df88796bd09a0d2ed292e59101478ad8 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda#69b8b6202a07720f448be700e300ccf4 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_1.conda#aec6c91c7371c26392a06708a73c70e5 https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.20-hd590300_0.conda#8e88f9389f1165d7c0936fe40d9a9a79 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda#e7ba12deb7020dd080c6c70e7b6f6a3d https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_5.conda#7a6bd7a12a4bd359e2afe6c0fa1acace +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-h43f5ff8_6.conda#e54a5ddc67e673f9105cf2a2e9c070b0 https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.0.0-hd590300_1.conda#ea25936bb4080d843790b586850f82b8 -https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.2-hd590300_1.conda#049b7df8bae5e184d1de42cdf64855f8 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.4.0-hd590300_0.conda#b26e8aa824079e1be0294e7152ca4559 https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda#97da8860a0da5413c7c98a3b3838a645 -https://conda.anaconda.org/conda-forge/linux-64/ninja-1.11.1-h924138e_0.conda#73a4953a2d9c115bdc10ff30a52f675f +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.0-h00ab1b0_0.conda#b048701d52e7cbb5f59ddd4d3b17bbf5 https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda#9d731343cff6ee2e5a25c4a091bf8e2a https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2#4b230e8381279d76131116660f5a241a @@ -32,9 +32,9 @@ https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161 https://conda.anaconda.org/conda-forge/linux-64/expat-2.6.2-h59595ed_0.conda#53fb86322bdb89496d7579fe3f02fd61 https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_1.conda#f07002e225d7a60a694d42a7bf5ff53f https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_1.conda#5fc11c6020d421960607d821310fcd4d -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_5.conda#e73e9cfd1191783392131e6238bdb3e9 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_6.conda#3666a850342f8f3be88f9a93d948d027 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.43-h2797004_0.conda#009981dd9cfcaa4dbfa25ffaed86bcae -https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.2-h2797004_0.conda#866983a220e27a80cb75e85cb30466a1 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda#b3316cbe90249da4f8e84cd66e1cc55b https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda#33277193f5b92bad9fdd230eb700929c https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc @@ -47,7 +47,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar. https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.27-pthreads_h413a1c8_0.conda#a356024784da6dfd4683dc5ecf45b155 https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.6.0-h1dd3fc0_3.conda#66f03896ffbe1a110ffda05c7a856504 https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-18.1.3-h4dfa4b3_0.conda#d39965123dffcad4d750989be65bcb7c -https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.45.2-h2c6b66d_0.conda#1423efca06ed343c1da0fc429bae0779 +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.45.3-h2c6b66d_0.conda#be7d70f2db41b674733667bdd69bd000 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.9-h8ee46fc_0.conda#077b6e8ad6a3ddb741fce2496dd01bec https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda#f27a24d46e3ea7b70a1f98e50c62508f https://conda.anaconda.org/conda-forge/linux-64/ccache-4.9.1-h1fcd64f_0.conda#3620f564bcf28c3524951b6f64f5c5ac @@ -72,7 +72,7 @@ https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py39h6dedee3_0.conda#557d64563e84ff21b14f586c7f662b7f https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda#248f521b64ce055e7feae3105e7abeb8 https://conda.anaconda.org/conda-forge/linux-64/pillow-10.3.0-py39h90a76f3_0.conda#799e6519cfffe2784db27b1db2ef33f3 -https://conda.anaconda.org/conda-forge/noarch/pluggy-1.4.0-pyhd8ed1ab_0.conda#139e9feb65187e916162917bb2484976 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.5.0-pyhd8ed1ab_0.conda#d3483c8fc2dc2cc3f5cf43e26d60cabf https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.1.2-pyhd8ed1ab_0.conda#b9a4dacf97241704529131a0dfc0494f https://conda.anaconda.org/conda-forge/noarch/pypy-7.3.15-1_pypy39.conda#a418a6c16bd6f7ed56b92194214791a0 https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda#7462280d81f639363e6e63c81276bd9e @@ -90,13 +90,13 @@ https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.4.0-pyhd8ed1 https://conda.anaconda.org/conda-forge/noarch/joblib-1.4.0-pyhd8ed1ab_0.conda#e0ed1bf13ce3a440e022157bf4764465 https://conda.anaconda.org/conda-forge/noarch/meson-1.4.0-pyhd8ed1ab_0.conda#52a0660cfa40b45bf254ecc3374cb2e0 https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda#f586ac1e56c8638b64f9c8122a7b8a67 -https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.7.1-pyhd8ed1ab_0.conda#dcb27826ffc94d5f04e241322239983b +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.8.0-pyhd8ed1ab_0.conda#573fe09d7bd0cd4bcc210d8369b5ca47 https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.4-pyhd8ed1ab_0.conda#a9d145de8c5f064b5fa68fb34725d9f4 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0-pyhd8ed1ab_0.conda#2cf4264fffb9e6eff6031c5b6884d61c https://conda.anaconda.org/conda-forge/linux-64/scipy-1.12.0-py39h6dedee3_2.conda#6c5d74bac41838f4377dfd45085e1fec https://conda.anaconda.org/conda-forge/linux-64/blas-2.122-openblas.conda#5065468105542a8b23ea47bd8b6fa55f https://conda.anaconda.org/conda-forge/noarch/importlib-resources-6.4.0-pyhd8ed1ab_0.conda#dcbadab7a68738a028e195ab68ab2d2e -https://conda.anaconda.org/conda-forge/noarch/meson-python-0.15.0-pyh0c530f3_0.conda#3bc64565ca78ce3bb80248d09926d8f9 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.16.0-pyh0c530f3_0.conda#e16f0dbf502da873be9f9adb0dc52547 https://conda.anaconda.org/conda-forge/linux-64/pyamg-5.1.0-py39h5fd064f_0.conda#04676d2a49da3cb608af77e04b796ce1 https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.5.0-pyhd8ed1ab_0.conda#d5f595da2daead898ca958ac62f0307b https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.4-py39h4e7d633_0.conda#58272019e595dde98d0844ae3ebf0cfe diff --git a/build_tools/azure/ubuntu_atlas_lock.txt b/build_tools/azure/ubuntu_atlas_lock.txt index c8e94493ccf28..f2110d2173a3c 100644 --- a/build_tools/azure/ubuntu_atlas_lock.txt +++ b/build_tools/azure/ubuntu_atlas_lock.txt @@ -35,7 +35,7 @@ pytest==7.4.4 # pytest-xdist pytest-xdist==3.5.0 # via -r build_tools/azure/ubuntu_atlas_requirements.txt -threadpoolctl==2.0.0 +threadpoolctl==3.1.0 # via -r build_tools/azure/ubuntu_atlas_requirements.txt tomli==2.0.1 # via diff --git a/build_tools/azure/ubuntu_atlas_requirements.txt b/build_tools/azure/ubuntu_atlas_requirements.txt index aab362dda0bf2..805d84d4d0aac 100644 --- a/build_tools/azure/ubuntu_atlas_requirements.txt +++ b/build_tools/azure/ubuntu_atlas_requirements.txt @@ -3,7 +3,7 @@ # build_tools/update_environments_and_lock_files.py cython==3.0.10 # min joblib==1.2.0 # min -threadpoolctl==2.0.0 # min +threadpoolctl==3.1.0 # min pytest<8 pytest-xdist ninja diff --git a/build_tools/circle/doc_linux-64_conda.lock b/build_tools/circle/doc_linux-64_conda.lock index 3f05468863ae6..60bbe8298c34c 100644 --- a/build_tools/circle/doc_linux-64_conda.lock +++ b/build_tools/circle/doc_linux-64_conda.lock @@ -318,5 +318,5 @@ https://conda.anaconda.org/conda-forge/noarch/sphinxext-opengraph-0.9.1-pyhd8ed1 # pip nbclient @ https://files.pythonhosted.org/packages/66/e8/00517a23d3eeaed0513e718fbc94aab26eaa1758f5690fc8578839791c79/nbclient-0.10.0-py3-none-any.whl#sha256=f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f # pip nbconvert @ https://files.pythonhosted.org/packages/23/8a/8d67cbd984739247e4b205c1143e2f71b25b4f71e180fe70f7cb2cf02633/nbconvert-7.16.3-py3-none-any.whl#sha256=ddeff14beeeedf3dd0bc506623e41e4507e551736de59df69a91f86700292b3b # pip jupyter-server @ https://files.pythonhosted.org/packages/07/46/6bb926b3bf878bf687b952fb6a4c09d014b4575a25960f2cd1a61793763f/jupyter_server-2.14.0-py3-none-any.whl#sha256=fb6be52c713e80e004fac34b35a0990d6d36ba06fd0a2b2ed82b899143a64210 -# pip jupyterlab-server @ https://files.pythonhosted.org/packages/6a/c9/b270a875916b18f137bb30ecd05031a2f05c95d47a8e8fbd3f805a72f593/jupyterlab_server-2.26.0-py3-none-any.whl#sha256=54622cbd330526a385ee0c1fdccdff3a1e7219bf3e864a335284a1270a1973df +# pip jupyterlab-server @ https://files.pythonhosted.org/packages/2f/b9/ed4ecad7cf1863a64920dc4c19b0376628b5d6bd28d2ec1e00cbac4ba2fb/jupyterlab_server-2.27.1-py3-none-any.whl#sha256=f5e26156e5258b24d532c84e7c74cc212e203bff93eb856f81c24c16daeecc75 # pip jupyterlite-sphinx @ https://files.pythonhosted.org/packages/38/c9/5f1142c005cf8d75830b10029e53f074324bc85cfca1f1d0f22a207b771c/jupyterlite_sphinx-0.9.3-py3-none-any.whl#sha256=be6332d16490ea2fa90b78187a2c5e1c357195966a25741d60b1790346571041 diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py index fd77cfd3c0721..8a4a30d235e0b 100644 --- a/build_tools/update_environments_and_lock_files.py +++ b/build_tools/update_environments_and_lock_files.py @@ -141,7 +141,10 @@ def remove_from(alist, to_remove): "folder": "build_tools/azure", "platform": "osx-64", "channel": "defaults", - "conda_dependencies": remove_from(common_dependencies, ["cython"]) + ["ccache"], + "conda_dependencies": remove_from( + common_dependencies, ["cython", "threadpoolctl"] + ) + + ["ccache"], "package_constraints": { "blas": "[build=mkl]", # scipy 1.12.x crashes on this platform (https://github.com/scipy/scipy/pull/20086) @@ -149,9 +152,9 @@ def remove_from(alist, to_remove): # channel. "scipy": "<1.12", }, - # TODO: put cython back to conda dependencies when required version is - # available on the main channel - "pip_dependencies": ["cython"], + # TODO: put cython and threadpoolctl back to conda dependencies when required + # version is available on the main channel + "pip_dependencies": ["cython", "threadpoolctl"], }, { "name": "pymin_conda_defaults_openblas", @@ -161,7 +164,8 @@ def remove_from(alist, to_remove): "platform": "linux-64", "channel": "defaults", "conda_dependencies": remove_from( - common_dependencies, ["pandas", "cython", "pip", "ninja", "meson-python"] + common_dependencies, + ["pandas", "cython", "threadpoolctl", "pip", "ninja", "meson-python"], ) + ["ccache"], "package_constraints": { @@ -170,12 +174,11 @@ def remove_from(alist, to_remove): "numpy": "1.21", # the min version is not available on the defaults channel "scipy": "1.7", # the min version has some low level crashes "matplotlib": "min", - "threadpoolctl": "2.2.0", "cython": "min", }, - # TODO: put cython back to conda dependencies when required version is - # available on the main channel - "pip_dependencies": ["cython"], + # TODO: put cython and threadpoolctl back to conda dependencies when required + # version is available on the main channel + "pip_dependencies": ["cython", "threadpoolctl"], }, { "name": "pymin_conda_forge_openblas_ubuntu_2204", @@ -381,7 +384,7 @@ def remove_from(alist, to_remove): ], "package_constraints": { "joblib": "min", - "threadpoolctl": "2.2.0", + "threadpoolctl": "3.1.0", "pytest": "min", "pytest-cov": "min", # no pytest-xdist because it causes issue on 32bit diff --git a/pyproject.toml b/pyproject.toml index 828569ecc71ee..69d9702716cb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "numpy>=1.19.5", "scipy>=1.6.0", "joblib>=1.2.0", - "threadpoolctl>=2.0.0", + "threadpoolctl>=3.1.0", ] requires-python = ">=3.9" license = {text = "new BSD"} @@ -45,7 +45,7 @@ tracker = "https://github.com/scikit-learn/scikit-learn/issues" [project.optional-dependencies] build = ["numpy>=1.19.5", "scipy>=1.6.0", "cython>=3.0.10", "meson-python>=0.15.0"] -install = ["numpy>=1.19.5", "scipy>=1.6.0", "joblib>=1.2.0", "threadpoolctl>=2.0.0"] +install = ["numpy>=1.19.5", "scipy>=1.6.0", "joblib>=1.2.0", "threadpoolctl>=3.1.0"] benchmark = ["matplotlib>=3.3.4", "pandas>=1.1.5", "memory_profiler>=0.57.0"] docs = [ "matplotlib>=3.3.4", diff --git a/sklearn/__init__.py b/sklearn/__init__.py index 30a3bbcdcbf66..45a26334a25f5 100644 --- a/sklearn/__init__.py +++ b/sklearn/__init__.py @@ -73,6 +73,15 @@ # We are not importing the rest of scikit-learn during the build # process, as it may not be compiled yet else: + # Import numpy, scipy to make sure that the BLAS libs are loaded before + # creating the ThreadpoolController. They would be imported just after + # when importing utils anyway. This makes it explicit and robust to changes + # in utils. + # (OpenMP is loaded by importing show_versions right after this block) + import numpy # noqa + import scipy.linalg # noqa + from threadpoolctl import ThreadpoolController + # `_distributor_init` allows distributors to run custom init code. # For instance, for the Windows wheel, this is used to pre-load the # vcomp shared library runtime for OpenMP embedded in the sklearn/.libs @@ -141,6 +150,12 @@ except ModuleNotFoundError: pass + # Set a global controller that can be used to locally limit the number of + # threads without looping through all shared libraries every time. + # This instantitation should not happen earlier because it needs all BLAS and + # OpenMP libs to be loaded first. + _threadpool_controller = ThreadpoolController() + def setup_module(module): """Fixture for the tests to assure globally controllable seeding of RNGs""" diff --git a/sklearn/_min_dependencies.py b/sklearn/_min_dependencies.py index a487a048c53c1..00315f31d4c3f 100644 --- a/sklearn/_min_dependencies.py +++ b/sklearn/_min_dependencies.py @@ -7,7 +7,7 @@ NUMPY_MIN_VERSION = "1.19.5" SCIPY_MIN_VERSION = "1.6.0" JOBLIB_MIN_VERSION = "1.2.0" -THREADPOOLCTL_MIN_VERSION = "2.0.0" +THREADPOOLCTL_MIN_VERSION = "3.1.0" PYTEST_MIN_VERSION = "7.1.2" CYTHON_MIN_VERSION = "3.0.10" diff --git a/sklearn/cluster/_kmeans.py b/sklearn/cluster/_kmeans.py index 16df12838d91f..2ab6f1e95563b 100644 --- a/sklearn/cluster/_kmeans.py +++ b/sklearn/cluster/_kmeans.py @@ -18,6 +18,7 @@ import numpy as np import scipy.sparse as sp +from .. import _threadpool_controller from ..base import ( BaseEstimator, ClassNamePrefixFeaturesOutMixin, @@ -31,7 +32,6 @@ from ..utils._openmp_helpers import _openmp_effective_n_threads from ..utils._param_validation import Interval, StrOptions, validate_params from ..utils.extmath import row_norms, stable_cumsum -from ..utils.fixes import threadpool_info, threadpool_limits from ..utils.sparsefuncs import mean_variance_axis from ..utils.sparsefuncs_fast import assign_rows_csr from ..utils.validation import ( @@ -622,6 +622,9 @@ def _kmeans_single_elkan( return labels, inertia, centers, i + 1 +# Threadpoolctl context to limit the number of threads in second level of +# nested parallelism (i.e. BLAS) to avoid oversubscription. +@_threadpool_controller.wrap(limits=1, user_api="blas") def _kmeans_single_lloyd( X, sample_weight, @@ -697,59 +700,56 @@ def _kmeans_single_lloyd( strict_convergence = False - # Threadpoolctl context to limit the number of threads in second level of - # nested parallelism (i.e. BLAS) to avoid oversubscription. - with threadpool_limits(limits=1, user_api="blas"): - for i in range(max_iter): - lloyd_iter( - X, - sample_weight, - centers, - centers_new, - weight_in_clusters, - labels, - center_shift, - n_threads, - ) + for i in range(max_iter): + lloyd_iter( + X, + sample_weight, + centers, + centers_new, + weight_in_clusters, + labels, + center_shift, + n_threads, + ) - if verbose: - inertia = _inertia(X, sample_weight, centers, labels, n_threads) - print(f"Iteration {i}, inertia {inertia}.") + if verbose: + inertia = _inertia(X, sample_weight, centers, labels, n_threads) + print(f"Iteration {i}, inertia {inertia}.") - centers, centers_new = centers_new, centers + centers, centers_new = centers_new, centers - if np.array_equal(labels, labels_old): - # First check the labels for strict convergence. + if np.array_equal(labels, labels_old): + # First check the labels for strict convergence. + if verbose: + print(f"Converged at iteration {i}: strict convergence.") + strict_convergence = True + break + else: + # No strict convergence, check for tol based convergence. + center_shift_tot = (center_shift**2).sum() + if center_shift_tot <= tol: if verbose: - print(f"Converged at iteration {i}: strict convergence.") - strict_convergence = True + print( + f"Converged at iteration {i}: center shift " + f"{center_shift_tot} within tolerance {tol}." + ) break - else: - # No strict convergence, check for tol based convergence. - center_shift_tot = (center_shift**2).sum() - if center_shift_tot <= tol: - if verbose: - print( - f"Converged at iteration {i}: center shift " - f"{center_shift_tot} within tolerance {tol}." - ) - break - labels_old[:] = labels + labels_old[:] = labels - if not strict_convergence: - # rerun E-step so that predicted labels match cluster centers - lloyd_iter( - X, - sample_weight, - centers, - centers, - weight_in_clusters, - labels, - center_shift, - n_threads, - update_centers=False, - ) + if not strict_convergence: + # rerun E-step so that predicted labels match cluster centers + lloyd_iter( + X, + sample_weight, + centers, + centers, + weight_in_clusters, + labels, + center_shift, + n_threads, + update_centers=False, + ) inertia = _inertia(X, sample_weight, centers, labels, n_threads) @@ -826,14 +826,10 @@ def _labels_inertia(X, sample_weight, centers, n_threads=1, return_inertia=True) return labels -def _labels_inertia_threadpool_limit( - X, sample_weight, centers, n_threads=1, return_inertia=True -): - """Same as _labels_inertia but in a threadpool_limits context.""" - with threadpool_limits(limits=1, user_api="blas"): - result = _labels_inertia(X, sample_weight, centers, n_threads, return_inertia) - - return result +# Same as _labels_inertia but in a threadpool_limits context. +_labels_inertia_threadpool_limit = _threadpool_controller.wrap( + limits=1, user_api="blas" +)(_labels_inertia) class _BaseKMeans( @@ -926,7 +922,7 @@ def _check_mkl_vcomp(self, X, n_samples): n_active_threads = int(np.ceil(n_samples / CHUNK_SIZE)) if n_active_threads < self._n_threads: - modules = threadpool_info() + modules = _threadpool_controller.info() has_vcomp = "vcomp" in [module["prefix"] for module in modules] has_mkl = ("mkl", "intel") in [ (module["internal_api"], module.get("threading_layer", None)) @@ -2148,7 +2144,7 @@ def fit(self, X, y=None, sample_weight=None): n_steps = (self.max_iter * n_samples) // self._batch_size - with threadpool_limits(limits=1, user_api="blas"): + with _threadpool_controller.limit(limits=1, user_api="blas"): # Perform the iterative optimization until convergence for i in range(n_steps): # Sample a minibatch from the full dataset @@ -2274,7 +2270,7 @@ def partial_fit(self, X, y=None, sample_weight=None): # Initialize number of samples seen since last reassignment self._n_since_last_reassign = 0 - with threadpool_limits(limits=1, user_api="blas"): + with _threadpool_controller.limit(limits=1, user_api="blas"): _mini_batch_step( X, sample_weight=sample_weight, diff --git a/sklearn/cluster/tests/test_k_means.py b/sklearn/cluster/tests/test_k_means.py index b6f718dce03e0..c3a41a65de632 100644 --- a/sklearn/cluster/tests/test_k_means.py +++ b/sklearn/cluster/tests/test_k_means.py @@ -8,6 +8,7 @@ import pytest from scipy import sparse as sp +from sklearn import _threadpool_controller from sklearn.base import clone from sklearn.cluster import KMeans, MiniBatchKMeans, k_means, kmeans_plusplus from sklearn.cluster._k_means_common import ( @@ -31,7 +32,7 @@ create_memmap_backed_data, ) from sklearn.utils.extmath import row_norms -from sklearn.utils.fixes import CSR_CONTAINERS, threadpool_limits +from sklearn.utils.fixes import CSR_CONTAINERS # non centered, sparse centers to check the centers = np.array( @@ -967,13 +968,13 @@ def test_result_equal_in_diff_n_threads(Estimator, global_random_seed): rnd = np.random.RandomState(global_random_seed) X = rnd.normal(size=(50, 10)) - with threadpool_limits(limits=1, user_api="openmp"): + with _threadpool_controller.limit(limits=1, user_api="openmp"): result_1 = ( Estimator(n_clusters=n_clusters, random_state=global_random_seed) .fit(X) .labels_ ) - with threadpool_limits(limits=2, user_api="openmp"): + with _threadpool_controller.limit(limits=2, user_api="openmp"): result_2 = ( Estimator(n_clusters=n_clusters, random_state=global_random_seed) .fit(X) diff --git a/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp index a686153c3ac9c..ef61158fedca8 100644 --- a/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp +++ b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp @@ -14,7 +14,7 @@ from numbers import Integral from scipy.sparse import issparse from ...utils import check_array, check_scalar from ...utils.fixes import _in_unstable_openblas_configuration -from ...utils.fixes import threadpool_limits +from ... import _threadpool_controller {{for name_suffix in ['64', '32']}} @@ -58,7 +58,7 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}): """ # Limit the number of threads in second level of nested parallelism for BLAS # to avoid threads over-subscription (in DOT or GEMM for instance). - with threadpool_limits(limits=1, user_api='blas'): + with _threadpool_controller.limit(limits=1, user_api='blas'): if metric in ("euclidean", "sqeuclidean"): # Specialized implementation of ArgKmin for the Euclidean distance # for the dense-dense and sparse-sparse cases. diff --git a/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp index f9719f6959dfc..b875499f44ed4 100644 --- a/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp +++ b/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp @@ -4,10 +4,10 @@ from libcpp.map cimport map as cpp_map, pair as cpp_pair from libc.stdlib cimport free from ...utils._typedefs cimport intp_t, float64_t +from ... import _threadpool_controller import numpy as np from scipy.sparse import issparse -from sklearn.utils.fixes import threadpool_limits from ._classmode cimport WeightingStrategy {{for name_suffix in ["32", "64"]}} @@ -66,7 +66,7 @@ cdef class ArgKminClassMode{{name_suffix}}(ArgKmin{{name_suffix}}): # Limit the number of threads in second level of nested parallelism for BLAS # to avoid threads over-subscription (in GEMM for instance). - with threadpool_limits(limits=1, user_api="blas"): + with _threadpool_controller.limit(limits=1, user_api="blas"): if pda.execute_in_parallel_on_Y: pda._parallel_on_Y() else: diff --git a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp index dcc97b4d32fd4..f4af378062bdc 100644 --- a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp +++ b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp @@ -17,7 +17,7 @@ from numbers import Real from scipy.sparse import issparse from ...utils import check_array, check_scalar from ...utils.fixes import _in_unstable_openblas_configuration -from ...utils.fixes import threadpool_limits +from ... import _threadpool_controller cnp.import_array() @@ -110,7 +110,7 @@ cdef class RadiusNeighbors{{name_suffix}}(BaseDistancesReduction{{name_suffix}}) # Limit the number of threads in second level of nested parallelism for BLAS # to avoid threads over-subscription (in GEMM for instance). - with threadpool_limits(limits=1, user_api="blas"): + with _threadpool_controller.limit(limits=1, user_api="blas"): if pda.execute_in_parallel_on_Y: pda._parallel_on_Y() else: diff --git a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp index 25067b43cd20c..ab12d7904c7fd 100644 --- a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp +++ b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp @@ -8,7 +8,7 @@ from ...utils._typedefs cimport intp_t, float64_t import numpy as np from scipy.sparse import issparse -from ...utils.fixes import threadpool_limits +from ... import _threadpool_controller {{for name_suffix in ["32", "64"]}} @@ -60,7 +60,7 @@ cdef class RadiusNeighborsClassMode{{name_suffix}}(RadiusNeighbors{{name_suffix} # Limit the number of threads in second level of nested parallelism for BLAS # to avoid threads over-subscription (in GEMM for instance). - with threadpool_limits(limits=1, user_api="blas"): + with _threadpool_controller.limit(limits=1, user_api="blas"): if pda.execute_in_parallel_on_Y: pda._parallel_on_Y() else: diff --git a/sklearn/metrics/tests/test_pairwise_distances_reduction.py b/sklearn/metrics/tests/test_pairwise_distances_reduction.py index e5983f9273d94..95dfa98178ee7 100644 --- a/sklearn/metrics/tests/test_pairwise_distances_reduction.py +++ b/sklearn/metrics/tests/test_pairwise_distances_reduction.py @@ -5,9 +5,9 @@ import numpy as np import pytest -import threadpoolctl from scipy.spatial.distance import cdist +from sklearn import _threadpool_controller from sklearn.metrics import euclidean_distances, pairwise_distances from sklearn.metrics._pairwise_distances_reduction import ( ArgKmin, @@ -1200,7 +1200,7 @@ def test_n_threads_agnosticism( **compute_parameters, ) - with threadpoolctl.threadpool_limits(limits=1, user_api="openmp"): + with _threadpool_controller.limit(limits=1, user_api="openmp"): dist, indices = Dispatcher.compute( X, Y, diff --git a/sklearn/utils/_show_versions.py b/sklearn/utils/_show_versions.py index 1431108477263..cc17b71b23799 100644 --- a/sklearn/utils/_show_versions.py +++ b/sklearn/utils/_show_versions.py @@ -9,8 +9,9 @@ import platform import sys +from threadpoolctl import threadpool_info + from .. import __version__ -from ..utils.fixes import threadpool_info from ._openmp_helpers import _openmp_parallelism_enabled diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index e33519a3154d8..21e62150b0356 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -18,7 +18,6 @@ import scipy import scipy.sparse.linalg import scipy.stats -import threadpoolctl import sklearn @@ -97,42 +96,6 @@ def _percentile(a, q, *, method="linear", **kwargs): from numpy import percentile # type: ignore # noqa -# compatibility fix for threadpoolctl >= 3.0.0 -# since version 3 it's possible to setup a global threadpool controller to avoid -# looping through all loaded shared libraries each time. -# the global controller is created during the first call to threadpoolctl. -def _get_threadpool_controller(): - if not hasattr(threadpoolctl, "ThreadpoolController"): - return None - - if not hasattr(sklearn, "_sklearn_threadpool_controller"): - sklearn._sklearn_threadpool_controller = threadpoolctl.ThreadpoolController() - - return sklearn._sklearn_threadpool_controller - - -def threadpool_limits(limits=None, user_api=None): - controller = _get_threadpool_controller() - if controller is not None: - return controller.limit(limits=limits, user_api=user_api) - else: - return threadpoolctl.threadpool_limits(limits=limits, user_api=user_api) - - -threadpool_limits.__doc__ = threadpoolctl.threadpool_limits.__doc__ - - -def threadpool_info(): - controller = _get_threadpool_controller() - if controller is not None: - return controller.info() - else: - return threadpoolctl.threadpool_info() - - -threadpool_info.__doc__ = threadpoolctl.threadpool_info.__doc__ - - # TODO: Remove when SciPy 1.11 is the minimum supported version def _mode(a, axis=0): if sp_version >= parse_version("1.9.0"): @@ -428,7 +391,7 @@ def _in_unstable_openblas_configuration(): import numpy # noqa import scipy # noqa - modules_info = threadpool_info() + modules_info = sklearn._threadpool_controller.info() open_blas_used = any(info["internal_api"] == "openblas" for info in modules_info) if not open_blas_used: diff --git a/sklearn/utils/tests/test_show_versions.py b/sklearn/utils/tests/test_show_versions.py index bd166dfd8e522..aade231e46f56 100644 --- a/sklearn/utils/tests/test_show_versions.py +++ b/sklearn/utils/tests/test_show_versions.py @@ -1,6 +1,7 @@ +from threadpoolctl import threadpool_info + from sklearn.utils._show_versions import _get_deps_info, _get_sys_info, show_versions from sklearn.utils._testing import ignore_warnings -from sklearn.utils.fixes import threadpool_info def test_get_sys_info():