scikit-learn · ogrisel · Nov 13, 2019 · Oct 10, 2019 · Oct 10, 2019 · Oct 10, 2019
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -48,7 +48,6 @@ jobs:
         DISTRIB: 'ubuntu'
         PYTHON_VERSION: '3.5'
         JOBLIB_VERSION: '0.11'
-        SKLEARN_NO_OPENMP: 'True'
       # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB
       py35_conda_openblas:
         DISTRIB: 'conda'
@@ -86,7 +85,6 @@ jobs:
         DISTRIB: 'ubuntu-32'
         PYTHON_VERSION: '3.5'
         JOBLIB_VERSION: '0.11'
-        SKLEARN_NO_OPENMP: 'True'
 
 - template: build_tools/azure/posix.yml
   parameters:
@@ -105,6 +103,19 @@ jobs:
         PYTEST_VERSION: '*'
         JOBLIB_VERSION: '*'
         COVERAGE: 'true'
+      pylatest_conda_mkl_no_openmp:
+        DISTRIB: 'conda'
+        PYTHON_VERSION: '*'
+        INSTALL_MKL: 'true'
+        NUMPY_VERSION: '*'
+        SCIPY_VERSION: '*'
+        CYTHON_VERSION: '*'
+        PILLOW_VERSION: '*'
+        PYTEST_VERSION: '*'
+        JOBLIB_VERSION: '*'
+        COVERAGE: 'true'
+        SKLEARN_TEST_NO_OPENMP: 'true'
+        SKLEARN_SKIP_OPENMP_TEST: 'true'
 
 - template: build_tools/azure/windows.yml
   parameters:

diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh
@@ -47,8 +47,11 @@ if [[ "$DISTRIB" == "conda" ]]; then
     fi
 
     if [[ "$UNAMESTR" == "Darwin" ]]; then
-        # on macOS, install an OpenMP-enabled clang/llvm from conda-forge
-        TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp"
+        if [[ "$SKLEARN_TEST_NO_OPENMP" != "true" ]]; then
+            # on macOS, install an OpenMP-enabled clang/llvm from conda-forge.
+            TO_INSTALL="$TO_INSTALL conda-forge::compilers \
+                        conda-forge::llvm-openmp"
+        fi
     fi
 
     # Old packages coming from the 'free' conda channel have been removed but

diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml
@@ -37,7 +37,6 @@ jobs:
         -e VIRTUALENV=testvenv
         -e JOBLIB_VERSION=$JOBLIB_VERSION
         -e PYTEST_VERSION=$PYTEST_VERSION
-        -e SKLEARN_NO_OPENMP=$SKLEARN_NO_OPENMP
         -e OMP_NUM_THREADS=$OMP_NUM_THREADS
         -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
         -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS

diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst
@@ -114,10 +114,12 @@ Building Scikit-learn also requires:
 
 .. note::
 
-   It is possible to build scikit-learn without OpenMP support by setting the
-   ``SKLEARN_NO_OPENMP`` environment variable (before cythonization). This is
-   not recommended since it will force some estimators to run in sequential
-   mode.
+   If OpenMP is not supported by the compiler, the build will be done with
+   OpenMP functionalities disabled. This is not recommended since it will force
+   some estimators to run in sequential mode instead of leveraging thread-based
+   parallelism. Setting the ``SKLEARN_FAIL_NO_OPENMP`` environment variable
+   (before cythonization) will force the build to fail if OpenMP is not
+   supported.
 
 Since version 0.21, scikit-learn automatically detects and use the linear
 algebrea library used by SciPy **at runtime**. Scikit-learn has therefore no

diff --git a/doc/developers/performance.rst b/doc/developers/performance.rst
@@ -332,16 +332,16 @@ memory alignment, direct blas calls...
 Using OpenMP
 ------------
 
-Since scikit-learn can be built without OpenMP support, it's necessary to
+Since scikit-learn can be built without OpenMP, it's necessary to
 protect each direct call to OpenMP. This can be done using the following
 syntax::
 
   # importing OpenMP
-  IF SKLEARN_OPENMP_SUPPORTED:
+  IF SKLEARN_OPENMP_PARALLELISM_ENABLED:
       cimport openmp
 
   # calling OpenMP
-  IF SKLEARN_OPENMP_SUPPORTED:
+  IF SKLEARN_OPENMP_PARALLELISM_ENABLED:
       max_threads = openmp.omp_get_max_threads()
   ELSE:
       max_threads = 1

diff --git a/setup.py b/setup.py
@@ -125,7 +125,7 @@ class build_ext_subclass(build_ext):
         def build_extensions(self):
             from sklearn._build_utils.openmp_helpers import get_openmp_flag
 
-            if not os.getenv('SKLEARN_NO_OPENMP'):
+            if sklearn._OPENMP_SUPPORTED:
                 openmp_flag = get_openmp_flag(self.compiler)
 
                 for e in self.extensions:

diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py
@@ -6,9 +6,12 @@
 
 
 import os
-from distutils.version import LooseVersion
+import sklearn
 import contextlib
 
+from distutils.version import LooseVersion
+
+from .pre_build_helpers import basic_check_build
 from .openmp_helpers import check_openmp_support
 
 
@@ -42,7 +45,24 @@ def cythonize_extensions(top_path, config):
     _check_cython_version()
     from Cython.Build import cythonize
 
-    with_openmp = check_openmp_support()
+    # Fast fail before cythonization if compiler fails compiling basic test
+    # code even without OpenMP
+    basic_check_build()
+
+    # check simple compilation with OpenMP. If it fails scikit-learn will be
+    # built without OpenMP and the test test_openmp_supported in the test suite
+    # will fail.
+    # `check_openmp_support` compiles a small test program to see if the
+    # compilers are properly configured to build with OpenMP. This is expensive
+    # and we only want to call this function once.
+    # The result of this check is cached as a private attribute on the sklearn
+    # module (only at build-time) to be used twice:
+    # - First to set the value of SKLEARN_OPENMP_PARALLELISM_ENABLED, the
+    #   cython build-time variable passed to the cythonize() call.
+    # - Then in the build_ext subclass defined in the top-level setup.py file
+    #   to actually build the compiled extensions with OpenMP flags if needed.
+    sklearn._OPENMP_SUPPORTED = check_openmp_support()
+
     n_jobs = 1
     with contextlib.suppress(ImportError):
         import joblib
@@ -55,7 +75,8 @@ def cythonize_extensions(top_path, config):
     config.ext_modules = cythonize(
         config.ext_modules,
         nthreads=n_jobs,
-        compile_time_env={'SKLEARN_OPENMP_SUPPORTED': with_openmp},
+        compile_time_env={
+            'SKLEARN_OPENMP_PARALLELISM_ENABLED': sklearn._OPENMP_SUPPORTED},
         compiler_directives={'language_level': 3})
 
 

diff --git a/sklearn/_build_utils/openmp_helpers.py b/sklearn/_build_utils/openmp_helpers.py
@@ -6,26 +6,13 @@
 
 import os
 import sys
-import glob
-import tempfile
 import textwrap
+import warnings
 import subprocess
 
-from numpy.distutils.ccompiler import new_compiler
-from distutils.sysconfig import customize_compiler
 from distutils.errors import CompileError, LinkError
 
-
-CCODE = textwrap.dedent(
-    """\
-    #include <omp.h>
-    #include <stdio.h>
-    int main(void) {
-    #pragma omp parallel
-    printf("nthreads=%d\\n", omp_get_num_threads());
-    return 0;
-    }
-    """)
+from .pre_build_helpers import compile_test_program
 
 
 def get_openmp_flag(compiler):
@@ -59,85 +46,69 @@ def get_openmp_flag(compiler):
 
 def check_openmp_support():
     """Check whether OpenMP test code can be compiled and run"""
-    ccompiler = new_compiler()
-    customize_compiler(ccompiler)
-
-    if os.getenv('SKLEARN_NO_OPENMP'):
-        # Build explicitly without OpenMP support
-        return False
-
-    start_dir = os.path.abspath('.')
-
-    with tempfile.TemporaryDirectory() as tmp_dir:
-        try:
-            os.chdir(tmp_dir)
-
-            # Write test program
-            with open('test_openmp.c', 'w') as f:
-                f.write(CCODE)
-
-            os.mkdir('objects')
+    code = textwrap.dedent(
+        """\
+        #include <omp.h>
+        #include <stdio.h>
+        int main(void) {
+        #pragma omp parallel
+        printf("nthreads=%d\\n", omp_get_num_threads());
+        return 0;
+        }
+        """)
 
-            # Compile, test program
-            openmp_flags = get_openmp_flag(ccompiler)
-            ccompiler.compile(['test_openmp.c'], output_dir='objects',
-                              extra_postargs=openmp_flags)
+    extra_preargs = os.getenv('LDFLAGS', None)
+    if extra_preargs is not None:
+        extra_preargs = extra_preargs.strip().split(" ")
+        extra_preargs = [
+            flag for flag in extra_preargs
+            if flag.startswith(('-L', '-Wl,-rpath', '-l'))]
 
-            # Link test program
-            extra_preargs = os.getenv('LDFLAGS', None)
-            if extra_preargs is not None:
-                extra_preargs = extra_preargs.strip().split(" ")
-                extra_preargs = [
-                    flag for flag in extra_preargs
-                    if flag.startswith(('-L', '-Wl,-rpath', '-l'))]
+    extra_postargs = get_openmp_flag
 
-            objects = glob.glob(
-                os.path.join('objects', '*' + ccompiler.obj_extension))
-            ccompiler.link_executable(objects, 'test_openmp',
+    try:
+        output = compile_test_program(code,
                                       extra_preargs=extra_preargs,
-                                      extra_postargs=openmp_flags)
-
-            # Run test program
-            output = subprocess.check_output('./test_openmp')
-            output = output.decode(sys.stdout.encoding or 'utf-8').splitlines()
-
-            # Check test program output
-            if 'nthreads=' in output[0]:
-                nthreads = int(output[0].strip().split('=')[1])
-                openmp_supported = (len(output) == nthreads)
-            else:
-                openmp_supported = False
+                                      extra_postargs=extra_postargs)
 
-        except (CompileError, LinkError, subprocess.CalledProcessError):
+        if 'nthreads=' in output[0]:
+            nthreads = int(output[0].strip().split('=')[1])
+            openmp_supported = len(output) == nthreads
+        else:
             openmp_supported = False
 
-        finally:
-            os.chdir(start_dir)
+    except (CompileError, LinkError, subprocess.CalledProcessError):
+        openmp_supported = False
 
-    err_message = textwrap.dedent(
-        """
-                            ***
+    if not openmp_supported:
+        if os.getenv("SKLEARN_FAIL_NO_OPENMP"):
+            raise CompileError("Failed to build with OpenMP")
+        else:
+            message = textwrap.dedent(
+                """
 
-        It seems that scikit-learn cannot be built with OpenMP support.
+                                ***********
+                                * WARNING *
+                                ***********
 
-        - Make sure you have followed the installation instructions:
+                It seems that scikit-learn cannot be built with OpenMP.
 
-            https://scikit-learn.org/dev/developers/advanced_installation.html
+                - Make sure you have followed the installation instructions:
 
-        - If your compiler supports OpenMP but the build still fails, please
-          submit a bug report at:
+                    https://scikit-learn.org/dev/developers/advanced_installation.html
 
-            https://github.com/scikit-learn/scikit-learn/issues
+                - If your compiler supports OpenMP but you still see this
+                  message, please submit a bug report at:
 
-        - If you want to build scikit-learn without OpenMP support, you can set
-          the environment variable SKLEARN_NO_OPENMP and rerun the build
-          command. Note however that some estimators will run in sequential
-          mode and their `n_jobs` parameter will have no effect anymore.
+                    https://github.com/scikit-learn/scikit-learn/issues
 
-                            ***
-        """)
+                - The build will continue with OpenMP-based parallelism
+                  disabled. Note however that some estimators will run in
+                  sequential mode instead of leveraging thread-based
+                  parallelism.
 
-    if not openmp_supported:
-        raise CompileError(err_message)
+                                    ***
+                """)
+            warnings.warn(message)
 
-    return True
+    return openmp_supported
diff --git a/sklearn/_build_utils/pre_build_helpers.py b/sklearn/_build_utils/pre_build_helpers.py
@@ -0,0 +1,70 @@
+"""Helpers to check build environment before actual build of scikit-learn"""
+
+import os
+import sys
+import glob
+import tempfile
+import textwrap
+import subprocess
+
+from distutils.sysconfig import customize_compiler
+from numpy.distutils.ccompiler import new_compiler
+
+
+def compile_test_program(code, extra_preargs=[], extra_postargs=[]):
+    """Check that some C code can be compiled and run"""
+    ccompiler = new_compiler()
+    customize_compiler(ccompiler)
+
+    # extra_(pre/post)args can be a callable to make it possible to get its
+    # value from the compiler
+    if callable(extra_preargs):
+        extra_preargs = extra_preargs(ccompiler)
+    if callable(extra_postargs):
+        extra_postargs = extra_postargs(ccompiler)
+
+    start_dir = os.path.abspath('.')
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        try:
+            os.chdir(tmp_dir)
+
+            # Write test program
+            with open('test_program.c', 'w') as f:
+                f.write(code)
+
+            os.mkdir('objects')
+
+            # Compile, test program
+            ccompiler.compile(['test_program.c'], output_dir='objects',
+                              extra_postargs=extra_postargs)
+
+            # Link test program
+            objects = glob.glob(
+                os.path.join('objects', '*' + ccompiler.obj_extension))
+            ccompiler.link_executable(objects, 'test_program',
+                                      extra_preargs=extra_preargs,
+                                      extra_postargs=extra_postargs)
+
+            # Run test program
+            # will raise a CalledProcessError if return code was non-zero
+            output = subprocess.check_output('./test_program')
+            output = output.decode(sys.stdout.encoding or 'utf-8').splitlines()
+        except Exception:
+            raise
+        finally:
+            os.chdir(start_dir)
+
+    return output
+
+
+def basic_check_build():
+    """Check basic compilation and linking of C code"""
+    code = textwrap.dedent(
+        """\
+        #include <stdio.h>
+        int main(void) {
+        return 0;
+        }
+        """)
+    compile_test_program(code)