scikit-learn · thomasjpfan · May 13, 2022 · Feb 9, 2022 · Feb 11, 2022 · Feb 11, 2022
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -63,7 +63,7 @@ jobs:
     matrix:
       pylatest_pip_scipy_dev:
         DISTRIB: 'conda-pip-scipy-dev'
-        PYTHON_VERSION: '*'
+        LOCK_FILE: './build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock'
         CHECK_WARNINGS: 'true'
         CHECK_PYTEST_SOFT_DEPENDENCY: 'true'
         TEST_DOCSTRINGS: 'true'
@@ -103,6 +103,7 @@ jobs:
     matrix:
       pylatest_pip_nogil:
         DISTRIB: 'pip-nogil'
+        LOCK_FILE: './build_tools/azure/python_nogil_lock.txt'
         COVERAGE: 'false'
 
 # Check compilation with intel C++ compiler (ICC)
@@ -122,9 +123,7 @@ jobs:
     matrix:
       pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
-        CONDA_CHANNEL: 'conda-forge'
-        PYTHON_VERSION: '*'
-        BLAS: 'mkl'
+        LOCK_FILE: 'build_tools/azure/pylatest_conda_forge_mkl_no_coverage_linux-64_conda.lock'
         COVERAGE: 'false'
         BUILD_WITH_ICC: 'true'
 
@@ -144,10 +143,9 @@ jobs:
       )
     matrix:
       pypy3:
-        DISTRIB: 'conda-mamba-pypy3'
-        DOCKER_CONTAINER: 'condaforge/mambaforge-pypy3:4.10.3-5'
-        PILLOW_VERSION: 'none'
-        PANDAS_VERSION: 'none'
+        DOCKER_CONTAINER: 'condaforge/miniforge3:4.10.3-5'
+        DISTRIB: 'conda-pypy3'
+        LOCK_FILE: './build_tools/azure/pypy3_linux-64_conda.lock'
 
 # Will run all the time regardless of linting outcome.
 - template: build_tools/azure/posix.yml
@@ -163,9 +161,7 @@ jobs:
     matrix:
       pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
-        CONDA_CHANNEL: 'conda-forge'
-        PYTHON_VERSION: '*'
-        BLAS: 'mkl'
+        LOCK_FILE: './build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock'
         COVERAGE: 'true'
         SHOW_SHORT_SUMMARY: 'true'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '42'  # default global random seed
@@ -184,9 +180,7 @@ jobs:
     matrix:
       py38_conda_forge_openblas_ubuntu_1804:
         DISTRIB: 'conda'
-        CONDA_CHANNEL: 'conda-forge'
-        PYTHON_VERSION: '3.8'
-        BLAS: 'openblas'
+        LOCK_FILE: './build_tools/azure/py38_conda_forge_openblas_ubuntu_1804_linux-64_conda.lock'
         COVERAGE: 'false'
         BUILD_WITH_ICC: 'false'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '0'  # non-default seed
@@ -207,30 +201,21 @@ jobs:
       # i.e. numpy 1.17.4 and scipy 1.3.3
       ubuntu_atlas:
         DISTRIB: 'ubuntu'
-        JOBLIB_VERSION: 'min'
-        PANDAS_VERSION: 'none'
-        THREADPOOLCTL_VERSION: 'min'
+        LOCK_FILE: './build_tools/azure/ubuntu_atlas_lock.txt'
         COVERAGE: 'false'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '1'  # non-default seed
       # Linux + Python 3.8 build with OpenBLAS
       py38_conda_defaults_openblas:
         DISTRIB: 'conda'
-        CONDA_CHANNEL: 'defaults'  # Anaconda main channel
-        PYTHON_VERSION: '3.8'
-        BLAS: 'openblas'
-        NUMPY_VERSION: 'min'
-        SCIPY_VERSION: 'min'
-        MATPLOTLIB_VERSION: 'min'
-        THREADPOOLCTL_VERSION: '2.2.0'
+        LOCK_FILE: './build_tools/azure/py38_conda_defaults_openblas_linux-64_conda.lock'
         SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1'
         SKLEARN_RUN_FLOAT32_TESTS: '1'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '2'  # non-default seed
       # Linux environment to test the latest available dependencies.
       # It runs tests requiring lightgbm, pandas and PyAMG.
       pylatest_pip_openblas_pandas:
         DISTRIB: 'conda-pip-latest'
-        PYTHON_VERSION: '3.9'
-        PYTEST_VERSION: '6.2.5'
+        LOCK_FILE: './build_tools/azure/pylatest_pip_openblas_pandas_linux-64_conda.lock'
         CHECK_PYTEST_SOFT_DEPENDENCY: 'true'
         TEST_DOCSTRINGS: 'true'
         CHECK_WARNINGS: 'true'
@@ -248,13 +233,11 @@ jobs:
       )
     matrix:
       debian_atlas_32bit:
-        DISTRIB: 'debian-32'
         DOCKER_CONTAINER: 'i386/debian:11.2'
-        JOBLIB_VERSION: 'min'
+        DISTRIB: 'debian-32'
+        LOCK_FILE: './build_tools/azure/debian_atlas_32bit_lock.txt'
         # disable pytest xdist due to unknown bug with 32-bit container
         PYTEST_XDIST_VERSION: 'none'
 if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then 
     TEST_CMD="$TEST_CMD -n$CPU_COUNT" 
 if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then 
     TEST_CMD="$TEST_CMD -n$CPU_COUNT" 
-        PYTEST_VERSION: 'min'
-        THREADPOOLCTL_VERSION: '2.2.0'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '4'  # non-default seed
 
 - template: build_tools/azure/posix.yml
@@ -270,12 +253,11 @@ jobs:
     matrix:
       pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
-        BLAS: 'mkl'
-        CONDA_CHANNEL: 'conda-forge'
+        LOCK_FILE: './build_tools/azure/pylatest_conda_forge_mkl_osx-64_conda.lock'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '5'  # non-default seed
       pylatest_conda_mkl_no_openmp:
         DISTRIB: 'conda'
-        BLAS: 'mkl'
+        LOCK_FILE: './build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock'
         SKLEARN_TEST_NO_OPENMP: 'true'
         SKLEARN_SKIP_OPENMP_TEST: 'true'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '6'  # non-default seed

diff --git a/build_tools/azure/debian_atlas_32bit_lock.txt b/build_tools/azure/debian_atlas_32bit_lock.txt
@@ -0,0 +1,34 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile --output-file=build_tools/azure/debian_atlas_32bit_lock.txt build_tools/azure/debian_atlas_32bit_requirements.txt
+#
+atomicwrites==1.4.0
+    # via pytest
+attrs==21.4.0
+    # via pytest
+cython==0.29.28
+    # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
+importlib-metadata==4.11.3
+    # via pytest
+joblib==1.0.0
+    # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
+more-itertools==8.13.0
+    # via pytest
+packaging==21.3
+    # via pytest
+pluggy==0.13.1
+    # via pytest
+py==1.11.0
+    # via pytest
+pyparsing==3.0.9
+    # via packaging
+pytest==5.0.1
+    # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
+threadpoolctl==2.2.0
+    # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
+wcwidth==0.2.5
+    # via pytest
+zipp==3.8.0
+    # via importlib-metadata
diff --git a/build_tools/azure/debian_atlas_32bit_requirements.txt b/build_tools/azure/debian_atlas_32bit_requirements.txt
@@ -0,0 +1,7 @@
+# DO NOT EDIT: this file is generated from the specification found in the
+# following script to centralize the configuration for all Azure CI builds:
+# build_tools/azure/update_environments_and_lock_files.py
+cython
+joblib==1.0.0  # min
+threadpoolctl==2.2.0
+pytest==5.0.1  # min
diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh
@@ -7,21 +7,8 @@ set -x
 source build_tools/shared.sh
 
 UNAMESTR=`uname`
-
 CCACHE_LINKS_DIR="/tmp/ccache"
 
-
-make_conda() {
-    TO_INSTALL="$@"
-    if [[ "$DISTRIB" == *"mamba"* ]]; then
-        mamba create -n $VIRTUALENV --yes $TO_INSTALL
-    else
-        conda config --show
-        conda create -n $VIRTUALENV --yes $TO_INSTALL
-    fi
-    source activate $VIRTUALENV
-}
-
 setup_ccache() {
     CCACHE_BIN=`which ccache || echo ""`
     if [[ "${CCACHE_BIN}" == "" ]]; then
@@ -53,8 +40,8 @@ pre_python_environment_install() {
                 python3-matplotlib libatlas3-base libatlas-base-dev \
                 python3-virtualenv python3-pandas ccache
 
-    elif [[ "$DISTRIB" == "conda-mamba-pypy3" ]]; then
-        # condaforge/mambaforge-pypy3 needs compilers
+    elif [[ "$DISTRIB" == "conda-pypy3" ]]; then
+        # need compilers
         apt-get -yq update
         apt-get -yq install build-essential
 
@@ -63,6 +50,14 @@ pre_python_environment_install() {
         sudo apt-get -yq update
         sudo apt-get install -yq ccache
         sudo apt-get build-dep -yq python3 python3-dev
+        setup_ccache  # speed-up the build of CPython itself
+        # build Python nogil
+        PYTHON_NOGIL_CLONE_PATH=../nogil
+        git clone --depth 1 https://github.com/colesbury/nogil $PYTHON_NOGIL_CLONE_PATH
+        cd $PYTHON_NOGIL_CLONE_PATH
+        ./configure && make -j 2
+        export PYTHON_NOGIL_PATH="${PYTHON_NOGIL_CLONE_PATH}/python"
+        cd $OLDPWD
 
     elif [[ "$BUILD_WITH_ICC" == "true" ]]; then
         wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
@@ -76,103 +71,35 @@ pre_python_environment_install() {
     fi
 }
 
-python_environment_install() {
-    if [[ "$DISTRIB" == "conda" || "$DISTRIB" == *"mamba"* ]]; then
-
-        if [[ "$CONDA_CHANNEL" != "" ]]; then
-            TO_INSTALL="--override-channels -c $CONDA_CHANNEL"
-        else
-            TO_INSTALL=""
-        fi
+python_environment_install_and_activate() {
+    if [[ "$DISTRIB" == "conda"* ]]; then
+        conda update -n base conda -y
+        # pin conda-lock to latest released version (needs manual update from time to time)
+        conda install -c conda-forge conda-lock==1.0.5 -y
+        conda-lock install --name $VIRTUALENV $LOCK_FILE
+        source activate $VIRTUALENV
 
-        if [[ "$DISTRIB" == *"pypy"* ]]; then
-            TO_INSTALL="$TO_INSTALL pypy"
-        else
-            TO_INSTALL="$TO_INSTALL python=$PYTHON_VERSION"
-        fi
-
-        TO_INSTALL="$TO_INSTALL ccache pip blas[build=$BLAS]"
-
-        TO_INSTALL="$TO_INSTALL $(get_dep numpy $NUMPY_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep scipy $SCIPY_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep cython $CYTHON_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep joblib $JOBLIB_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep pandas $PANDAS_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep pyamg $PYAMG_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep Pillow $PILLOW_VERSION)"
-        TO_INSTALL="$TO_INSTALL $(get_dep matplotlib $MATPLOTLIB_VERSION)"
-
-        if [[ "$UNAMESTR" == "Darwin" ]] && [[ "$SKLEARN_TEST_NO_OPENMP" != "true" ]]; then
-                TO_INSTALL="$TO_INSTALL compilers llvm-openmp"
-        fi
-
-        make_conda $TO_INSTALL
-
-    elif [[ "$DISTRIB" == "ubuntu" ]] || [[ "$DISTRIB" == "debian-32" ]]; then
+    elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then
         python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
         source $VIRTUALENV/bin/activate
+        pip install -r "${LOCK_FILE}"
 
-        python -m pip install $(get_dep cython $CYTHON_VERSION) \
-                $(get_dep joblib $JOBLIB_VERSION)
-
-    elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
-        # Since conda main channel usually lacks behind on the latest releases,
-        # we use pypi to test against the latest releases of the dependencies.
-        # conda is still used as a convenient way to install Python and pip.
-        make_conda "ccache python=$PYTHON_VERSION"
-        python -m pip install -U pip
-
-        python -m pip install pandas matplotlib scikit-image pyamg
-        # do not install dependencies for lightgbm since it requires scikit-learn.
-        python -m pip install "lightgbm>=3.0.0" --no-deps
+    elif [[ "$DISTRIB" == "pip-nogil" ]]; then
+        ${PYTHON_NOGIL_PATH} -m venv $VIRTUALENV
+        source $VIRTUALENV/bin/activate
+        pip install -r "${LOCK_FILE}"
+    fi
 
-    elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
-        make_conda "ccache python=$PYTHON_VERSION"
-        python -m pip install -U pip
-        echo "Installing numpy and scipy master wheels"
+    if [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
+        echo "Installing development dependency wheels"
         dev_anaconda_url=https://pypi.anaconda.org/scipy-wheels-nightly/simple
         pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy pandas scipy
+        echo "Installing Cython from PyPI enabling pre-releases"
         pip install --pre cython
         echo "Installing joblib master"
         pip install https://github.com/joblib/joblib/archive/master.zip
         echo "Installing pillow master"
         pip install https://github.com/python-pillow/Pillow/archive/main.zip
-
-    elif [[ "$DISTRIB" == "pip-nogil" ]]; then
-        setup_ccache  # speed-up the build of CPython it-self
-        ORIGINAL_FOLDER=`pwd`
-        cd ..
-        git clone --depth 1 https://github.com/colesbury/nogil
-        cd nogil
-        ./configure && make -j 2
-        ./python -m venv $ORIGINAL_FOLDER/$VIRTUALENV
-        cd $ORIGINAL_FOLDER
-        source $VIRTUALENV/bin/activate
-
-        python -m pip install -U pip
-        # The pip version that comes with the nogil branch of CPython
-        # automatically uses the custom nogil index as its highest priority
-        # index to fetch patched versions of libraries with native code that
-        # would otherwise depend on the GIL.
-        echo "Installing build dependencies with pip from the nogil repository: https://d1yxz45j0ypngg.cloudfront.net/"
-        pip install numpy scipy cython joblib threadpoolctl
-
-    fi
-
-    python -m pip install $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \
-            $(get_dep pytest $PYTEST_VERSION) \
-            $(get_dep pytest-xdist $PYTEST_XDIST_VERSION)
-
-    if [[ "$COVERAGE" == "true" ]]; then
-        # XXX: coverage is temporary pinned to 6.2 because 6.3 is not fork-safe
-        # cf. https://github.com/nedbat/coveragepy/issues/1310
-        python -m pip install codecov pytest-cov coverage==6.2
-    fi
-
-    if [[ "$TEST_DOCSTRINGS" == "true" ]]; then
-        # numpydoc requires sphinx
-        python -m pip install sphinx
-        python -m pip install numpydoc
     fi
 }
 
@@ -184,7 +111,7 @@ scikit_learn_install() {
     # workers with 2 cores when building the compiled extensions of scikit-learn.
     export SKLEARN_BUILD_PARALLEL=3
 
-    if [[ "$UNAMESTR" == "Darwin" ]] && [[ "$SKLEARN_TEST_NO_OPENMP" == "true" ]]; then
+    if [[ "$UNAMESTR" == "Darwin" && "$SKLEARN_TEST_NO_OPENMP" == "true" ]]; then
         # Without openmp, we use the system clang. Here we use /usr/bin/ar
         # instead because llvm-ar errors
         export AR=/usr/bin/ar
@@ -220,7 +147,7 @@ scikit_learn_install() {
 
 main() {
     pre_python_environment_install
-    python_environment_install
+    python_environment_install_and_activate
     scikit_learn_install
 }