From d46c1b54e2f21245cee545ca09032ff161e3c868 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 4 Mar 2022 15:04:12 -0500 Subject: [PATCH 1/4] BLD Monkeypatch windows build to stablize build --- setup.py | 9 ++ sklearn/externals/_numpy_complier_patch.py | 139 +++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 sklearn/externals/_numpy_complier_patch.py diff --git a/setup.py b/setup.py index 8c1665cd8ba98..74e2ced8f3580 100755 --- a/setup.py +++ b/setup.py @@ -307,6 +307,15 @@ def setup_package(): # may use numpy.distutils compiler classes. from numpy.distutils.core import setup + # Monkeypatchs CCompiler.spawn to prevent random wheel build errors + # https://github.com/scikit-learn/scikit-learn/issues/22310 + # https://github.com/numpy/numpy/pull/20640 + from numpy.distutils.ccompiler import replace_method + from distutils.ccompiler import CCompiler + from sklearn.externals._numpy_complier_patch import CCompiler_spawn # noqa + + replace_method(CCompiler, "spawn", CCompiler_spawn) + metadata["configuration"] = configuration setup(**metadata) diff --git a/sklearn/externals/_numpy_complier_patch.py b/sklearn/externals/_numpy_complier_patch.py new file mode 100644 index 0000000000000..a424d8e99a8ef --- /dev/null +++ b/sklearn/externals/_numpy_complier_patch.py @@ -0,0 +1,139 @@ +# Copyright (c) 2005-2022, NumPy Developers. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. + +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import sys +import subprocess +import re +from distutils.errors import DistutilsExecError + +from numpy.distutils import log + + +def is_sequence(seq): + if isinstance(seq, str): + return False + try: + len(seq) + except Exception: + return False + return True + + +def forward_bytes_to_stdout(val): + """ + Forward bytes from a subprocess call to the console, without attempting to + decode them. + + The assumption is that the subprocess call already returned bytes in + a suitable encoding. + """ + if hasattr(sys.stdout, "buffer"): + # use the underlying binary output if there is one + sys.stdout.buffer.write(val) + elif hasattr(sys.stdout, "encoding"): + # round-trip the encoding if necessary + sys.stdout.write(val.decode(sys.stdout.encoding)) + else: + # make a best-guess at the encoding + sys.stdout.write(val.decode("utf8", errors="replace")) + + +def CCompiler_spawn(self, cmd, display=None, env=None): + """ + Execute a command in a sub-process. + + Parameters + ---------- + cmd : str + The command to execute. + display : str or sequence of str, optional + The text to add to the log file kept by `numpy.distutils`. + If not given, `display` is equal to `cmd`. + env: a dictionary for environment variables, optional + + Returns + ------- + None + + Raises + ------ + DistutilsExecError + If the command failed, i.e. the exit status was not 0. + + """ + env = env if env is not None else dict(os.environ) + if display is None: + display = cmd + if is_sequence(display): + display = " ".join(list(display)) + log.info(display) + try: + if self.verbose: + subprocess.check_output(cmd, env=env) + else: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env) + except subprocess.CalledProcessError as exc: + o = exc.output + s = exc.returncode + except OSError as e: + # OSError doesn't have the same hooks for the exception + # output, but exec_command() historically would use an + # empty string for EnvironmentError (base class for + # OSError) + # o = b'' + # still that would make the end-user lost in translation! + o = f"\n\n{e}\n\n\n" + try: + o = o.encode(sys.stdout.encoding) + except AttributeError: + o = o.encode("utf8") + # status previously used by exec_command() for parent + # of OSError + s = 127 + else: + # use a convenience return here so that any kind of + # caught exception will execute the default code after the + # try / except block, which handles various exceptions + return None + + if is_sequence(cmd): + cmd = " ".join(list(cmd)) + + if self.verbose: + forward_bytes_to_stdout(o) + + if re.search(b"Too many open files", o): + msg = "\nTry rerunning setup command until build succeeds." + else: + msg = "" + raise DistutilsExecError( + 'Command "%s" failed with exit status %d%s' % (cmd, s, msg) + ) From 51300773ada0e99b342b05af81ff5353f7a859d5 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 4 Mar 2022 15:08:00 -0500 Subject: [PATCH 2/4] CI [cd build] From c860c25ea31b369df5217475ef4a28662da9357c Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 18 Mar 2022 13:45:37 -0400 Subject: [PATCH 3/4] DOC Add more links [cd build] --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 74e2ced8f3580..29690d5b8c3ed 100755 --- a/setup.py +++ b/setup.py @@ -307,7 +307,9 @@ def setup_package(): # may use numpy.distutils compiler classes. from numpy.distutils.core import setup - # Monkeypatchs CCompiler.spawn to prevent random wheel build errors + # Monkeypatchs CCompiler.spawn to prevent random wheel build errors on Windows + # The build errors on Windows was because msvccompiler spawn was not threadsafe + # https://github.com/pypa/distutils/issues/5 # https://github.com/scikit-learn/scikit-learn/issues/22310 # https://github.com/numpy/numpy/pull/20640 from numpy.distutils.ccompiler import replace_method From 2a2ab160cb3a95c4a0c390e6157117a419d1a42c Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Sat, 19 Mar 2022 22:20:26 -0400 Subject: [PATCH 4/4] CLN Address comments --- setup.py | 5 +++-- .../{_numpy_complier_patch.py => _numpy_compiler_patch.py} | 0 2 files changed, 3 insertions(+), 2 deletions(-) rename sklearn/externals/{_numpy_complier_patch.py => _numpy_compiler_patch.py} (100%) diff --git a/setup.py b/setup.py index 29690d5b8c3ed..7ad32e95e53a5 100755 --- a/setup.py +++ b/setup.py @@ -307,14 +307,15 @@ def setup_package(): # may use numpy.distutils compiler classes. from numpy.distutils.core import setup - # Monkeypatchs CCompiler.spawn to prevent random wheel build errors on Windows + # Monkeypatches CCompiler.spawn to prevent random wheel build errors on Windows # The build errors on Windows was because msvccompiler spawn was not threadsafe + # This fixed can be removed when we build with numpy >= 1.22.2 on Windows. # https://github.com/pypa/distutils/issues/5 # https://github.com/scikit-learn/scikit-learn/issues/22310 # https://github.com/numpy/numpy/pull/20640 from numpy.distutils.ccompiler import replace_method from distutils.ccompiler import CCompiler - from sklearn.externals._numpy_complier_patch import CCompiler_spawn # noqa + from sklearn.externals._numpy_compiler_patch import CCompiler_spawn replace_method(CCompiler, "spawn", CCompiler_spawn) diff --git a/sklearn/externals/_numpy_complier_patch.py b/sklearn/externals/_numpy_compiler_patch.py similarity index 100% rename from sklearn/externals/_numpy_complier_patch.py rename to sklearn/externals/_numpy_compiler_patch.py