scikit-learn · glemaitre · Oct 25, 2021 · Sep 8, 2021 · Sep 15, 2021 · Sep 15, 2021
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -114,8 +114,12 @@ jobs:
       image: ubuntu-2004:202101-01
     resource_class: arm.medium
     environment:
+      # Use the latest supported version of python
+      - PYTHON_VERSION: '3.9'
       - OMP_NUM_THREADS: 2
       - OPENBLAS_NUM_THREADS: 2
+      - NUMPY_VERSION: 'latest'
+      - SCIPY_VERSION: 'latest'
       - CYTHON_VERSION: 'latest'
       - JOBLIB_VERSION: 'latest'
       - THREADPOOLCTL_VERSION: 'latest'
@@ -126,18 +130,16 @@ jobs:
       - checkout
       - run: ./build_tools/circle/checkout_merge_commit.sh
       - restore_cache:
-          key: v1-datasets-{{ .Branch }}
+          key: linux-arm64-{{ .Branch }}
       - run: ./build_tools/circle/build_test_arm.sh
       - save_cache:
-          key: doc-ccache-{{ .Branch }}-{{ .BuildNum }}
+          key: linux-arm64-{{ .Branch }}
           paths:
-            - ~/.ccache
+            - ~/.cache/ccache
             - ~/.cache/pip
-      - save_cache:
-          key: v1-datasets-{{ .Branch }}
-          paths:
             - ~/scikit_learn_data
-
+            # The source build folder.
+            - ~/project/build
   deploy:
     docker:
       - image: circleci/python:3.7

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -16,3 +16,6 @@
 
 # PR 20412: Use experimental_string_processing=true in Black
 3ae7c7615343bbd36acece57825d8b0d70fd9da4
+
+# PR 20502: Runs Black on examples
+70a185ae59b4362633d18b0d0083abb1b6f7370c
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1 @@
-/doc/whats_new.rst merge=union
+/doc/whats_new/v*.rst merge=union
diff --git a/PULL_REQUEST_TEMPLATE.md → .github/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md → .github/PULL_REQUEST_TEMPLATE.md
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -124,6 +124,8 @@ jobs:
 
       - name: Setup Python
         uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'  # update once build dependencies are available
 
       - name: Build source distribution
         run: bash build_tools/github/build_source.sh

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -23,6 +23,7 @@ recursive-exclude benchmarks *
 recursive-exclude .binder *
 recursive-exclude .circleci *
 exclude .codecov.yml
+exclude .git-blame-ignore-revs
 exclude .mailmap
 exclude .pre-commit-config.yaml
 exclude azure-pipelines.yml

diff --git a/README.rst b/README.rst
@@ -20,8 +20,8 @@
 .. |PythonVersion| image:: https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9-blue
 .. _PythonVersion: https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9-blue
 
-.. |PyPi| image:: https://badge.fury.io/py/scikit-learn.svg
-.. _PyPi: https://badge.fury.io/py/scikit-learn
+.. |PyPi| image:: https://img.shields.io/pypi/v/scikit-learn
+.. _PyPi: https://pypi.org/project/scikit-learn
 
 .. |Black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
 .. _Black: https://github.com/psf/black
@@ -35,13 +35,13 @@
 .. |SciPyMinVersion| replace:: 1.1.0
 .. |JoblibMinVersion| replace:: 0.11
 .. |ThreadpoolctlMinVersion| replace:: 2.0.0
-.. |MatplotlibMinVersion| replace:: 2.2.2
+.. |MatplotlibMinVersion| replace:: 2.2.3
 .. |Scikit-ImageMinVersion| replace:: 0.14.5
 .. |PandasMinVersion| replace:: 0.25.0
 .. |SeabornMinVersion| replace:: 0.9.0
 .. |PytestMinVersion| replace:: 5.0.1
 
-.. image:: doc/logos/scikit-learn-logo.png
+.. image:: https://raw.githubusercontent.com/scikit-learn/scikit-learn/main/doc/logos/scikit-learn-logo.png
   :target: https://scikit-learn.org/
 
 **scikit-learn** is a Python module for machine learning built on top of

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -97,8 +97,9 @@ jobs:
         )
       )
     matrix:
-      pylatest_conda_mkl:
+      pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
+        CONDA_CHANNEL: 'conda-forge'
         PYTHON_VERSION: '*'
         BLAS: 'mkl'
         COVERAGE: 'false'
@@ -116,8 +117,9 @@ jobs:
         not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]'))
       )
     matrix:
-      pylatest_conda_mkl:
+      pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
+        CONDA_CHANNEL: 'conda-forge'
         PYTHON_VERSION: '*'
         BLAS: 'mkl'
         COVERAGE: 'true'
@@ -135,8 +137,9 @@ jobs:
         ne(variables['Build.Reason'], 'Schedule')
       )
     matrix:
-      py37_conda:
+      py37_conda_forge_openblas_ubuntu_1804:
         DISTRIB: 'conda'
+        CONDA_CHANNEL: 'conda-forge'
         PYTHON_VERSION: '3.7'
         BLAS: 'openblas'
         COVERAGE: 'false'
@@ -164,14 +167,15 @@ jobs:
         THREADPOOLCTL_VERSION: 'min'
         COVERAGE: 'false'
       # Linux + Python 3.7 build with OpenBLAS and without SITE_JOBLIB
-      py37_conda_openblas:
+      py37_conda_defaults_openblas:
         DISTRIB: 'conda'
+        CONDA_CHANNEL: 'defaults'  # Anaconda main channel
         PYTHON_VERSION: '3.7'
         BLAS: 'openblas'
         NUMPY_VERSION: 'min'
         SCIPY_VERSION: 'min'
         MATPLOTLIB_VERSION: 'min'
-        THREADPOOLCTL_VERSION: '2.0.0'
+        THREADPOOLCTL_VERSION: '2.2.0'
       # Linux environment to test the latest available dependencies and MKL.
       # It runs tests requiring lightgbm, pandas and PyAMG.
       pylatest_pip_openblas_pandas:
@@ -206,7 +210,7 @@ jobs:
         # disable pytest xdist due to unknown bug with 32-bit container
         PYTEST_XDIST_VERSION: 'none'
         PYTEST_VERSION: 'min'
-        THREADPOOLCTL_VERSION: 'min'
+        THREADPOOLCTL_VERSION: '2.2.0'
 
 - template: build_tools/azure/posix.yml
   parameters:
@@ -242,13 +246,14 @@ jobs:
         ne(variables['Build.Reason'], 'Schedule')
       )
     matrix:
-      py37_conda_mkl:
+      py37_conda_forge_mkl:
+        DISTRIB: 'conda'
+        CONDA_CHANNEL: 'conda-forge'
         PYTHON_VERSION: '3.7'
         CHECK_WARNINGS: 'true'
         PYTHON_ARCH: '64'
         PYTEST_VERSION: '*'
         COVERAGE: 'true'
-        DISTRIB: 'conda'
       py37_pip_openblas_32bit:
         PYTHON_VERSION: '3.7'
         PYTHON_ARCH: '32'
diff --git a/benchmarks/bench_hist_gradient_boosting.py b/benchmarks/bench_hist_gradient_boosting.py
@@ -140,7 +140,9 @@ def one_run(n_samples):
     lightgbm_score_duration = None
     if args.lightgbm:
         print("Fitting a LightGBM model...")
-        lightgbm_est = get_equivalent_estimator(est, lib="lightgbm")
+        lightgbm_est = get_equivalent_estimator(
+            est, lib="lightgbm", n_classes=args.n_classes
+        )
 
         tic = time()
         lightgbm_est.fit(X_train, y_train, sample_weight=sample_weight_train)

diff --git a/benchmarks/bench_hist_gradient_boosting_threading.py b/benchmarks/bench_hist_gradient_boosting_threading.py
@@ -135,7 +135,9 @@ def get_estimator_and_data():
     for libname in ["lightgbm", "xgboost", "catboost"]:
         if getattr(args, libname):
             print(libname)
-            est = get_equivalent_estimator(sklearn_est, lib=libname)
+            est = get_equivalent_estimator(
+                sklearn_est, lib=libname, n_classes=args.n_classes
+            )
             pprint(est.get_params())
 
 
@@ -169,7 +171,9 @@ def one_run(n_threads, n_samples):
     lightgbm_score_duration = None
     if args.lightgbm:
         print("Fitting a LightGBM model...")
-        lightgbm_est = get_equivalent_estimator(est, lib="lightgbm")
+        lightgbm_est = get_equivalent_estimator(
+            est, lib="lightgbm", n_classes=args.n_classes
+        )
         lightgbm_est.set_params(num_threads=n_threads)
 
         tic = time()

diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh
@@ -16,6 +16,7 @@ make_conda() {
     if [[ "$DISTRIB" == *"mamba"* ]]; then
         mamba create -n $VIRTUALENV --yes $TO_INSTALL
     else
+        conda config --show
         conda create -n $VIRTUALENV --yes $TO_INSTALL
     fi
     source activate $VIRTUALENV
@@ -38,7 +39,7 @@ source build_tools/shared.sh
 if [[ "$DISTRIB" == "conda" || "$DISTRIB" == *"mamba"* ]]; then
 
     if [[ "$CONDA_CHANNEL" != "" ]]; then
-        TO_INSTALL="-c $CONDA_CHANNEL"
+        TO_INSTALL="--override-channels -c $CONDA_CHANNEL"
     else
         TO_INSTALL=""
     fi
@@ -113,9 +114,8 @@ elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
     python -m pip install --only-binary :all: scikit-image || true
 
     python -m pip install pandas matplotlib pyamg
-    # do not install dependencies for lightgbm since it requires scikit-learn
-    # and install a version less than 3.0.0 until the issue #18316 is solved.
-    python -m pip install "lightgbm<3.0.0" --no-deps
+    # do not install dependencies for lightgbm since it requires scikit-learn.
+    python -m pip install "lightgbm>=3.0.0" --no-deps
 elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
     # FIXME: temporary fix to link against system libraries on linux
     export LDFLAGS="$LDFLAGS -Wl,--sysroot=/"
@@ -129,7 +129,7 @@ elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
     echo "Installing joblib master"
     pip install https://github.com/joblib/joblib/archive/master.zip
     echo "Installing pillow master"
-    pip install https://github.com/python-pillow/Pillow/archive/master.zip
+    pip install https://github.com/python-pillow/Pillow/archive/main.zip
 fi
 
 python -m pip install $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \

diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh
@@ -12,18 +12,20 @@ if [[ "$BUILD_WITH_ICC" == "true" ]]; then
     source /opt/intel/oneapi/setvars.sh
 fi
 
-python --version
-python -c "import numpy; print('numpy %s' % numpy.__version__)"
-python -c "import scipy; print('scipy %s' % scipy.__version__)"
-python -c "\
-try:
-    import pandas
-    print('pandas %s' % pandas.__version__)
-except ImportError:
-    print('pandas not installed')
-"
-python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())"
-pip list
+mkdir -p $TEST_DIR
+cp setup.cfg $TEST_DIR
+cd $TEST_DIR
+
+python -c "import sklearn; sklearn.show_versions()"
+python -m threadpoolctl -i sklearn
+
+if ! command -v conda &> /dev/null
+then
+    pip list
+else
+    # conda list provides more info than pip list (when available)
+    conda list
+fi
 
 TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML"
 
@@ -46,10 +48,6 @@ if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then
     TEST_CMD="$TEST_CMD -n2"
 fi
 
-mkdir -p $TEST_DIR
-cp setup.cfg $TEST_DIR
-cd $TEST_DIR
-
 set -x
 $TEST_CMD --pyargs sklearn
 set +x
diff --git a/build_tools/circle/build_test_arm.sh b/build_tools/circle/build_test_arm.sh
@@ -4,6 +4,8 @@ set -e
 set -x
 
 UNAMESTR=`uname`
+N_CORES=`nproc --all`
+
 
 setup_ccache() {
     echo "Setting up ccache"
@@ -12,49 +14,75 @@ setup_ccache() {
     for name in gcc g++ cc c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++; do
       ln -s $(which ccache) "/tmp/ccache/${name}"
     done
-    export PATH="/tmp/ccache/:${PATH}"
-    ccache -M 256M
+    export PATH="/tmp/ccache:${PATH}"
+    # Unset ccache limits
+    ccache -F 0
+    ccache -M 0
 }
 
 # imports get_dep
 source build_tools/shared.sh
 
 sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
 sudo apt-get update
-sudo apt-get install python3-virtualenv ccache
-python3 -m virtualenv --system-site-packages --python=python3 testenv
-source testenv/bin/activate
-pip install --upgrade pip
+
+# Setup conda environment
+MINICONDA_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh"
+
+# Install Mambaforge
+wget $MINICONDA_URL -O mambaforge.sh
+MINICONDA_PATH=$HOME/miniconda
+chmod +x mambaforge.sh && ./mambaforge.sh -b -p $MINICONDA_PATH
+export PATH=$MINICONDA_PATH/bin:$PATH
+mamba init --all --verbose
+mamba update --yes conda
+
+# Create environment and install dependencies
+mamba create -n testenv --yes $(get_dep python $PYTHON_VERSION)
+source activate testenv
+
+# Use the latest by default
+mamba install --verbose -y  ccache \
+                            pip \
+                            $(get_dep numpy $NUMPY_VERSION) \
+                            $(get_dep scipy $SCIPY_VERSION) \
+                            $(get_dep cython $CYTHON_VERSION) \
+                            $(get_dep joblib $JOBLIB_VERSION) \
+                            $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \
+                            $(get_dep pytest $PYTEST_VERSION) \
+                            $(get_dep pytest-xdist $PYTEST_XDIST_VERSION)
 setup_ccache
-python -m pip install $(get_dep cython $CYTHON_VERSION) \
-                      $(get_dep joblib $JOBLIB_VERSION)
-python -m pip install $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \
-                      $(get_dep pytest $PYTEST_VERSION) \
-                      $(get_dep pytest-xdist $PYTEST_XDIST_VERSION)
 
 if [[ "$COVERAGE" == "true" ]]; then
-    python -m pip install codecov pytest-cov
-fi
-
-if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then
-    python -m pip install pytest-xdist
+    mamba install --verbose -y codecov pytest-cov
 fi
 
 if [[ "$TEST_DOCSTRINGS" == "true" ]]; then
     # numpydoc requires sphinx
-    python -m pip install sphinx
-    python -m pip install numpydoc
+    mamba install --verbose -y sphinx
+    mamba install --verbose -y numpydoc
 fi
 
 python --version
 
-# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
-# workers with 2 cores when building the compiled extensions of scikit-learn.
-export SKLEARN_BUILD_PARALLEL=3
+# Set parallelism to $N_CORES + 1 to overlap IO bound tasks with CPU bound tasks on CI
+# workers with $N_CORES cores when building the compiled extensions of scikit-learn.
+export SKLEARN_BUILD_PARALLEL=$(($N_CORES + 1))
+
+# Disable the build isolation and build in the tree so that the same folder can be
+# cached between CI runs.
+# TODO: remove the '--use-feature' flag when made obsolete in pip 21.3.
+pip install --verbose --no-build-isolation --use-feature=in-tree-build .
+
+# Report cache usage
+ccache -s --verbose
+
+mamba list
 
-python -m pip list
-pip install --verbose --editable .
-ccache -s
+# Changing directory not to have module resolution use scikit-learn source
+# directory but to the installed package.
+cd /tmp
 python -c "import sklearn; sklearn.show_versions()"
 python -m threadpoolctl --import sklearn
-python -m pytest sklearn
+# Test using as many workers as available cores
+pytest --pyargs -n $N_CORES sklearn
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		/doc/whats_new.rst merge=union
		/doc/whats_new/v*.rst merge=union