From b81dff6e77367330e1ffa560db689b713e2bdb92 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 18 Dec 2019 18:27:16 +0100 Subject: [PATCH 01/12] Include vcomp140.dll in windows wheels --- .appveyor.yml | 2 ++ appveyor/vendor_vcomp140.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 appveyor/vendor_vcomp140.py diff --git a/.appveyor.yml b/.appveyor.yml index 3a58477c..3014c751 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -102,6 +102,8 @@ install: # Install the build and runtime dependencies of the project. - "%CMD_IN_ENV% pip install --timeout=60 numpy==%NP_BUILD_DEP%" - "%CMD_IN_ENV% pip install --timeout=60 -r ../appveyor/requirements.txt" + - "%CMD_IN_ENV% python setup.py build" + - "%CMD_IN_ENV% python ../appveyor/vendor_vcomp140.py" - "%CMD_IN_ENV% python setup.py bdist_wheel" - ps: "ls dist" - "%CMD_IN_ENV% twine check dist/*" diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py new file mode 100644 index 00000000..7838c30a --- /dev/null +++ b/appveyor/vendor_vcomp140.py @@ -0,0 +1,31 @@ +import os +import shutil +from glob import glob + +VCOMP140_SRC_PATH = "C:\\Windows\System32\\vcomp140.dll" +TARGET_FOLDER_GLOB_PATTERN = "build/lib.*/sklearn/utils" + + +def main(): + # TODO: use threadpoolctl to locate vcomp140.dll instead? + if not os.path.exists(VCOMP140_SRC_PATH): + raise ValueError("Could not find %r" % VCOMP140_SRC_PATH) + + if not os.path.isdir("build"): + raise RuntimeError("Could not find ./build/ folder. " + "Run 'python setup.py build' first") + target_folders = glob(TARGET_FOLDER_GLOB_PATTERN) + if len(target_folders) == 0: + raise RuntimeError("Could not find folder matching '%s'" + % TARGET_FOLDER_GLOB_PATTERN) + if len(target_folders) > 1: + raise RuntimeError("Found too many target folders: %r" + % target_folders) + target_folder = target_folders[0] + + print("Copying %r to %r" % (VCOMP140_SRC_PATH, target_folder)) + shutil.copy2(VCOMP140_SRC_PATH, target_folder) + + +if __name__ == "__main__": + main() From 95b843dcd38ffa76210655012f380c1d2804109c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 18 Dec 2019 20:52:56 +0100 Subject: [PATCH 02/12] Try to collect artifacts --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 3014c751..8b4fd03e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -130,7 +130,7 @@ test_script: artifacts: # Archive the generated wheel package in the ci.appveyor.com build report. - - path: dist\* + - path: 'scikit-learn\dist\*' on_success: # Upload the generated wheel package to Rackspace From bbc2fae3ea0b304d84933359e42fd3f4c941c910 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 13:16:09 +0100 Subject: [PATCH 03/12] New strategy for vcomp140.dll pre-loading --- .appveyor.yml | 2 +- .gitmodules | 2 +- .travis.yml | 2 +- appveyor/vendor_vcomp140.py | 54 ++++++++++++++++++++++++++++++++++--- scikit-learn | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8b4fd03e..b06af4e6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,7 @@ environment: global: - BUILD_COMMIT: 0.22 + BUILD_COMMIT: distributor-init-first # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter # See: http://stackoverflow.com/a/13751649/163740 diff --git a/.gitmodules b/.gitmodules index e00dfbb9..7ffb843b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "scikit-learn"] path = scikit-learn - url = git://github.com/scikit-learn/scikit-learn.git + url = git://github.com/ogrisel/scikit-learn.git [submodule "multibuild"] path = multibuild url = https://github.com/matthew-brett/multibuild.git diff --git a/.travis.yml b/.travis.yml index 29d155d0..ad18110f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: - REPO_DIR="scikit-learn" - - BUILD_COMMIT=0.22 + - BUILD_COMMIT=distributor-init-first - PLAT=x86_64 - UNICODE_WIDTH=32 - NP_BUILD_DEP="numpy==1.11.0" diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index 7838c30a..dcb20c64 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -1,13 +1,53 @@ import os +import os.path as op import shutil from glob import glob +import textwrap VCOMP140_SRC_PATH = "C:\\Windows\System32\\vcomp140.dll" -TARGET_FOLDER_GLOB_PATTERN = "build/lib.*/sklearn/utils" +TARGET_FOLDER_GLOB_PATTERN = "build/lib.*/sklearn" + + +def make_distributor_init(dirname): + ''' + Create a _distributor_init.py file for OpenBlas + ''' + with open(op.join(dirname, '_distributor_init.py'), 'wt') as f: + f.write(textwrap.dedent(""" + ''' + Helper to preload windows dlls to prevent "dll not found" errors. + Once a DLL is preloaded, its namespace is made available to any + subsequent DLL. This file originated in the scikit-learn-wheels + github repo, and is created as part of the scripts that build the + wheel. + ''' + import os + import os.path as op + from ctypes import WinDLL + from glob import glob + if os.name == 'nt': + # convention for storing / loading the DLL from + # sklearn/.libs/, if present + DLL_filenames = [] + basedir = op.dirname(__file__) + libs_dir = op.abspath(op.join(basedir, '.libs')) + + # Pre-load the DLL and warn if more than 1 is found. + for filename in glob(op.join(libs_dir,'vcomp*dll')): + WinDLL(op.abspath(filename)) + DLL_filenames.append(filename) + if len(DLL_filenames) > 1: + import warnings + warnings.warn("loaded more than 1 DLL from .libs:\\n%s" % + "\\n".join(DLL_filenames), + stacklevel=1) + """)) def main(): - # TODO: use threadpoolctl to locate vcomp140.dll instead? + # TODO: use threadpoolctl to dynamically locate the right vcomp dll + # instead? This would require first in-place building scikit-learn + # to make it "importable". if not os.path.exists(VCOMP140_SRC_PATH): raise ValueError("Could not find %r" % VCOMP140_SRC_PATH) @@ -21,11 +61,19 @@ def main(): if len(target_folders) > 1: raise RuntimeError("Found too many target folders: %r" % target_folders) - target_folder = target_folders[0] + target_folder = op.join(target_folders[0], ".libs") + + # create the "sklearn/.libs" subfolder + if not op.exists(target_folder): + os.mkdir(target_folder) print("Copying %r to %r" % (VCOMP140_SRC_PATH, target_folder)) shutil.copy2(VCOMP140_SRC_PATH, target_folder) + # Generate the _distributor_init file in the source tree. + print("Generating the sklearn/_distributor_init.py file") + make_distributor_init("sklearn") + if __name__ == "__main__": main() diff --git a/scikit-learn b/scikit-learn index 7389dbac..d274cc64 160000 --- a/scikit-learn +++ b/scikit-learn @@ -1 +1 @@ -Subproject commit 7389dbac82d362f296dc2746f10e43ffa1615660 +Subproject commit d274cc64bba90495087251e8e8cad4c00609cede From 1e3c0c348c78ce149e4e577d02000ef948e15af0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 14:49:44 +0100 Subject: [PATCH 04/12] Simplify distributor init script --- appveyor/vendor_vcomp140.py | 60 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index dcb20c64..cf2caced 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -1,3 +1,10 @@ +"""Embed the vcomp dll before generating the scikit-lern Windows wheel. + +This script should be run from the root of the scikit-learn source tree, +after running the `python setup.py build` command and before running +the `python setup.py bdist_wheel` command. +""" + import os import os.path as op import shutil @@ -8,14 +15,18 @@ TARGET_FOLDER_GLOB_PATTERN = "build/lib.*/sklearn" -def make_distributor_init(dirname): - ''' - Create a _distributor_init.py file for OpenBlas - ''' - with open(op.join(dirname, '_distributor_init.py'), 'wt') as f: +def make_distributor_init(sklearn_dirname, dll_filename): + """Create a _distributor_init.py file for the vcomp dll. + + This file is imported first when importing the sklearn package so as + to pre-load the vendored vcomp dll. + """ + distributor_init = op.join(sklearn_dirname, '_distributor_init.py') + with open(distributor_init, 'wt') as f: f.write(textwrap.dedent(""" ''' - Helper to preload windows dlls to prevent "dll not found" errors. + Helper to preload the OpenMP dll to prevent "dll not found" + errors. Once a DLL is preloaded, its namespace is made available to any subsequent DLL. This file originated in the scikit-learn-wheels github repo, and is created as part of the scripts that build the @@ -24,24 +35,15 @@ def make_distributor_init(dirname): import os import os.path as op from ctypes import WinDLL - from glob import glob + + if os.name == 'nt': - # convention for storing / loading the DLL from - # sklearn/.libs/, if present - DLL_filenames = [] - basedir = op.dirname(__file__) - libs_dir = op.abspath(op.join(basedir, '.libs')) - - # Pre-load the DLL and warn if more than 1 is found. - for filename in glob(op.join(libs_dir,'vcomp*dll')): - WinDLL(op.abspath(filename)) - DLL_filenames.append(filename) - if len(DLL_filenames) > 1: - import warnings - warnings.warn("loaded more than 1 DLL from .libs:\\n%s" % - "\\n".join(DLL_filenames), - stacklevel=1) - """)) + # Pre-load the DLL stored in sklearn/.libs by convention. + dll_path = op.join(op.dirname(__file__), '.libs', '{}') + WinDLL(op.abspath(dll_path)) + + """.format(dll_filename))) + return op.abspath(distributor_init) def main(): @@ -59,20 +61,20 @@ def main(): raise RuntimeError("Could not find folder matching '%s'" % TARGET_FOLDER_GLOB_PATTERN) if len(target_folders) > 1: - raise RuntimeError("Found too many target folders: %r" - % target_folders) - target_folder = op.join(target_folders[0], ".libs") + raise RuntimeError("Found too many target folders: '%s'" + % "', '".join(target_folders)) + target_folder = op.abspath(op.join(target_folders[0], ".libs")) # create the "sklearn/.libs" subfolder if not op.exists(target_folder): os.mkdir(target_folder) - print("Copying %r to %r" % (VCOMP140_SRC_PATH, target_folder)) + print("Copying '%s' to:\n%s" % (VCOMP140_SRC_PATH, target_folder)) shutil.copy2(VCOMP140_SRC_PATH, target_folder) # Generate the _distributor_init file in the source tree. - print("Generating the sklearn/_distributor_init.py file") - make_distributor_init("sklearn") + print("Generating the '_distributor_init.py' file in:") + print(make_distributor_init("sklearn", op.basename(VCOMP140_SRC_PATH))) if __name__ == "__main__": From 2e91056f60292b11faec94354278cb790248fd87 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 19:25:13 +0100 Subject: [PATCH 05/12] Update appveyor/vendor_vcomp140.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ci skip] Co-Authored-By: Jérémie du Boisberranger <34657725+jeremiedbb@users.noreply.github.com> --- appveyor/vendor_vcomp140.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index cf2caced..bfa767b5 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -1,4 +1,4 @@ -"""Embed the vcomp dll before generating the scikit-lern Windows wheel. +"""Embed the vcomp dll before generating the scikit-learn Windows wheel. This script should be run from the root of the scikit-learn source tree, after running the `python setup.py build` command and before running From 175064a4be67c7118093f953680f77c6c0c2ab0f Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 19:25:24 +0100 Subject: [PATCH 06/12] Update appveyor/vendor_vcomp140.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ci skip] Co-Authored-By: Jérémie du Boisberranger <34657725+jeremiedbb@users.noreply.github.com> --- appveyor/vendor_vcomp140.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index bfa767b5..e0d1bf25 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -53,7 +53,7 @@ def main(): if not os.path.exists(VCOMP140_SRC_PATH): raise ValueError("Could not find %r" % VCOMP140_SRC_PATH) - if not os.path.isdir("build"): + if not op.isdir("build"): raise RuntimeError("Could not find ./build/ folder. " "Run 'python setup.py build' first") target_folders = glob(TARGET_FOLDER_GLOB_PATTERN) From f4e4846816140807c6e4fd1843ee89a4e8f7f7ea Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 19:25:43 +0100 Subject: [PATCH 07/12] Update appveyor/vendor_vcomp140.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ci skip] Co-Authored-By: Jérémie du Boisberranger <34657725+jeremiedbb@users.noreply.github.com> --- appveyor/vendor_vcomp140.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index e0d1bf25..56dbdaca 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -50,7 +50,7 @@ def main(): # TODO: use threadpoolctl to dynamically locate the right vcomp dll # instead? This would require first in-place building scikit-learn # to make it "importable". - if not os.path.exists(VCOMP140_SRC_PATH): + if not op.exists(VCOMP140_SRC_PATH): raise ValueError("Could not find %r" % VCOMP140_SRC_PATH) if not op.isdir("build"): From 6e68a765e8b8d07f0ac7a0aa46313ab7e0e97009 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 19 Dec 2019 20:40:27 +0100 Subject: [PATCH 08/12] Fix indent [ci skip] --- appveyor/vendor_vcomp140.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/vendor_vcomp140.py b/appveyor/vendor_vcomp140.py index 56dbdaca..f06bd0d7 100644 --- a/appveyor/vendor_vcomp140.py +++ b/appveyor/vendor_vcomp140.py @@ -42,7 +42,7 @@ def make_distributor_init(sklearn_dirname, dll_filename): dll_path = op.join(op.dirname(__file__), '.libs', '{}') WinDLL(op.abspath(dll_path)) - """.format(dll_filename))) + """.format(dll_filename))) return op.abspath(distributor_init) From f42ec46bc7c72bfbd09e2679a73fe6d4be77c6ce Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sat, 21 Dec 2019 00:34:28 +0100 Subject: [PATCH 09/12] Target the main repo (master branch for now) --- .appveyor.yml | 2 +- .gitmodules | 2 +- .travis.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b06af4e6..63767b8a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,7 @@ environment: global: - BUILD_COMMIT: distributor-init-first + BUILD_COMMIT: master # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter # See: http://stackoverflow.com/a/13751649/163740 diff --git a/.gitmodules b/.gitmodules index 7ffb843b..e00dfbb9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "scikit-learn"] path = scikit-learn - url = git://github.com/ogrisel/scikit-learn.git + url = git://github.com/scikit-learn/scikit-learn.git [submodule "multibuild"] path = multibuild url = https://github.com/matthew-brett/multibuild.git diff --git a/.travis.yml b/.travis.yml index ad18110f..a045a975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: - REPO_DIR="scikit-learn" - - BUILD_COMMIT=distributor-init-first + - BUILD_COMMIT=master - PLAT=x86_64 - UNICODE_WIDTH=32 - NP_BUILD_DEP="numpy==1.11.0" From 0eea0c98e7f09fffb57096f24cdec73c4fa47a35 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 2 Jan 2020 12:22:03 +0100 Subject: [PATCH 10/12] Test 0.22.1-release branch --- .appveyor.yml | 2 +- .gitmodules | 2 +- .travis.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 63767b8a..ad95b010 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,7 @@ environment: global: - BUILD_COMMIT: master + BUILD_COMMIT: 0.22.1-release-branch # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter # See: http://stackoverflow.com/a/13751649/163740 diff --git a/.gitmodules b/.gitmodules index e00dfbb9..7ffb843b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "scikit-learn"] path = scikit-learn - url = git://github.com/scikit-learn/scikit-learn.git + url = git://github.com/ogrisel/scikit-learn.git [submodule "multibuild"] path = multibuild url = https://github.com/matthew-brett/multibuild.git diff --git a/.travis.yml b/.travis.yml index a045a975..3ad52b53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: - REPO_DIR="scikit-learn" - - BUILD_COMMIT=master + - BUILD_COMMIT=0.22.1-release-branch - PLAT=x86_64 - UNICODE_WIDTH=32 - NP_BUILD_DEP="numpy==1.11.0" From b592afb364693aecbed6953b60c32dea09411378 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 2 Jan 2020 12:44:14 +0100 Subject: [PATCH 11/12] Point back to master branch --- scikit-learn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scikit-learn b/scikit-learn index d274cc64..c8c21ae1 160000 --- a/scikit-learn +++ b/scikit-learn @@ -1 +1 @@ -Subproject commit d274cc64bba90495087251e8e8cad4c00609cede +Subproject commit c8c21ae18a7289d8ae9f837946e1e7f85b0337b8 From 1316356cc64c75692ef852baaa65f28d4e7c892d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 2 Jan 2020 15:52:14 +0100 Subject: [PATCH 12/12] Build release 0.22.1 --- .appveyor.yml | 2 +- .gitmodules | 2 +- .travis.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ad95b010..61af3be4 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,7 @@ environment: global: - BUILD_COMMIT: 0.22.1-release-branch + BUILD_COMMIT: 0.22.1 # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter # See: http://stackoverflow.com/a/13751649/163740 diff --git a/.gitmodules b/.gitmodules index 7ffb843b..e00dfbb9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "scikit-learn"] path = scikit-learn - url = git://github.com/ogrisel/scikit-learn.git + url = git://github.com/scikit-learn/scikit-learn.git [submodule "multibuild"] path = multibuild url = https://github.com/matthew-brett/multibuild.git diff --git a/.travis.yml b/.travis.yml index 3ad52b53..9fbe9ec5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: - REPO_DIR="scikit-learn" - - BUILD_COMMIT=0.22.1-release-branch + - BUILD_COMMIT=0.22.1 - PLAT=x86_64 - UNICODE_WIDTH=32 - NP_BUILD_DEP="numpy==1.11.0"