scikit-learn · betatim · Jun 10, 2024 · Jun 5, 2024 · Jun 5, 2024 · Jun 5, 2024
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -62,40 +62,26 @@ jobs:
         SKLEARN_WARNINGS_AS_ERRORS: '1'
         CHECK_PYTEST_SOFT_DEPENDENCY: 'true'
 
-- template: build_tools/azure/posix-docker.yml
-  # Experimental CPython branch without the Global Interpreter Lock:
-  # https://github.com/colesbury/nogil/
-  #
-  # The nogil build relies on a dedicated PyPI-style index to install patched
-  # versions of NumPy, SciPy and Cython maintained by @colesbury and that
-  # include specific fixes to make them run correctly without relying on the GIL.
-  #
-  # The goal of this CI entry is to make sure that we do not introduce any
-  # dependency on the GIL in scikit-learn itself. An auxiliary goal is to early
-  # detect any regression in the patched build dependencies to report them
-  # upstream. The long-term goal is to be able to stop having to maintain
-  # multiprocessing based workaround / hacks in joblib / loky to make multi-CPU
-  # computing in scikit-learn efficient by default using regular threads.
-  #
-  # If this experimental entry becomes too unstable, feel free to disable it.
+- template: build_tools/azure/posix.yml
+  # CPython 3.13 free-threaded build
   parameters:
-    name: Linux_nogil
-    vmImage: ubuntu-20.04
+    name: Linux_free_threaded
+    vmImage: ubuntu-22.04
     dependsOn: [git_commit, linting]
     condition: |
       and(
         succeeded(),
         not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
         or(eq(variables['Build.Reason'], 'Schedule'),
-           contains(dependencies['git_commit']['outputs']['commit.message'], '[nogil]'
+           contains(dependencies['git_commit']['outputs']['commit.message'], '[free-threaded]'
           )
         )
       )
     matrix:
-      pylatest_pip_nogil:
-        DOCKER_CONTAINER: 'nogil/python'
-        DISTRIB: 'pip-nogil'
-        LOCK_FILE: './build_tools/azure/python_nogil_lock.txt'
+      pylatest_pip_free_threaded:
+        PYTHON_GIL: '0'
+        DISTRIB: 'pip-free-threaded'
+        LOCK_FILE: './build_tools/azure/cpython_free_threaded_lock.txt'
         COVERAGE: 'false'
 
 - job: Linux_Nightly_Pyodide

diff --git a/build_tools/azure/cpython_free_threaded_lock.txt b/build_tools/azure/cpython_free_threaded_lock.txt
@@ -0,0 +1,39 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=/scikit-learn/build_tools/azure/cpython_free_threaded_lock.txt /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+#
+execnet==2.1.1
+    # via pytest-xdist
+iniconfig==2.0.0
+    # via pytest
+joblib==1.4.2
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+meson==1.4.1
+    # via meson-python
+meson-python==0.16.0
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+ninja==1.11.1.1
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+packaging==24.0
+    # via
+    #   meson-python
+    #   pyproject-metadata
+    #   pytest
+pluggy==1.5.0
+    # via pytest
+pyproject-metadata==0.8.0
+    # via meson-python
+pytest==8.2.2
+    # via
+    #   -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+    #   pytest-xdist
+pytest-xdist==3.6.1
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+threadpoolctl==3.5.0
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==70.0.0
+    # via -r /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt
diff --git a/build_tools/azure/cpython_free_threaded_requirements.txt b/build_tools/azure/cpython_free_threaded_requirements.txt
@@ -0,0 +1,17 @@
+# To generate cpython_free_threaded_lock.txt, use the following command:
+# docker run -v $PWD:/scikit-learn -it ubuntu bash -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -yq update; apt-get install software-properties-common ccache -y; add-apt-repository --yes ppa:deadsnakes/nightly; apt-get update -y; apt-get install -y --no-install-recommends python3.13-dev python3.13-venv python3.13-nogil; python3.13t -m venv /venvs/myenv; source /venvs/myenv/bin/activate; pip install pip-tools; pip-compile --allow-unsafe /scikit-learn/build_tools/azure/cpython_free_threaded_requirements.txt -o /scikit-learn/build_tools/azure/cpython_free_threaded_lock.txt'
+
+# The reason behind it is that you need python-3.13t to generate the pip lock
+# file. For pure Python wheel this does not really matter. But when there are
+# cython, numpy and scipy releases that have a CPython 3.13 free-threaded
+# wheel, we can add them here and this is important that the Python 3.13
+# free-threaded wheel is picked up in the lock-file
+joblib
+threadpoolctl
+pytest
+pytest-xdist
+ninja
+meson-python
+# For some reason some of our tests require setuptools.
+# TODO: update those tests to remove the dependency.
+setuptools
diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh
@@ -38,6 +38,17 @@ pre_python_environment_install() {
         apt-get install -y python3-dev python3-numpy python3-scipy \
                 python3-matplotlib libatlas3-base libatlas-base-dev \
                 python3-virtualenv python3-pandas ccache git
+
+    # TODO for now we use CPython 3.13 from Ubuntu deadsnakes PPA. When CPython
+    # 3.13 is released (scheduled October 2024) we can use something more
+    # similar to other conda+pip based builds
+    elif [[ "$DISTRIB" == "pip-free-threaded" ]]; then
+        sudo apt-get -yq update
+        sudo apt-get install -yq ccache
+        sudo apt-get install -yq software-properties-common
+        sudo add-apt-repository --yes ppa:deadsnakes/nightly
+        sudo apt-get update -yq
+        sudo apt-get install -yq --no-install-recommends python3.13-dev python3.13-venv python3.13-nogil
     fi
 }
 
@@ -66,10 +77,25 @@ python_environment_install_and_activate() {
         source $VIRTUALENV/bin/activate
         pip install -r "${LOCK_FILE}"
 
-    elif [[ "$DISTRIB" == "pip-nogil" ]]; then
-        python -m venv $VIRTUALENV
+    elif [[ "$DISTRIB" == "pip-free-threaded" ]]; then
+        python3.13t -m venv $VIRTUALENV
         source $VIRTUALENV/bin/activate
         pip install -r "${LOCK_FILE}"
+        # TODO for now need pip 24.1b1 to find free-threaded wheels
+        pip install -U --pre pip
+        # TODO When there are CPython 3.13 free-threaded wheels for numpy and
+        # scipy move this to
+        # build_tools/azure/cpython_free_threaded_requirements.txt. For now we
+        # install them from scientific-python-nightly-wheels
+        dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+        dev_packages="numpy scipy"
+        pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url $dev_packages
+        # TODO Move cython to
+        # build_tools/azure/cpython_free_threaded_requirements.txt when there
+        # is a CPython 3.13 free-threaded wheel
+        # For now, we need the development version of Cython which has CPython
+        # 3.13 free-threaded fixes so we install it from source
+        pip install git+https://github.com/cython/cython
     fi
 
     if [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
@@ -86,11 +112,6 @@ python_environment_install_and_activate() {
         pip install https://github.com/joblib/joblib/archive/master.zip
         echo "Installing pillow from latest sources"
         pip install https://github.com/python-pillow/Pillow/archive/main.zip
-
-    elif [[ "$DISTRIB" == "pip-nogil" ]]; then
-        apt-get -yq update
-        apt-get install -yq ccache
-
     fi
 }
 

diff --git a/build_tools/azure/python_nogil_lock.txt b/build_tools/azure/python_nogil_lock.txt
diff --git a/build_tools/azure/python_nogil_requirements.txt b/build_tools/azure/python_nogil_requirements.txt
diff --git a/build_tools/azure/test_docs.sh b/build_tools/azure/test_docs.sh
@@ -4,7 +4,7 @@ set -e
 
 if [[ "$DISTRIB" =~ ^conda.* ]]; then
     source activate $VIRTUALENV
-elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "pip-nogil" ]]; then
+elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "pip-free-threaded" ]]; then
     source $VIRTUALENV/bin/activate
 fi
 

diff --git a/build_tools/shared.sh b/build_tools/shared.sh
@@ -29,7 +29,7 @@ show_installed_libraries(){
 activate_environment() {
     if [[ "$DISTRIB" =~ ^conda.* ]]; then
         source activate $VIRTUALENV
-    elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" || "$DISTRIB" == "pip-nogil" ]]; then
+    elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" || "$DISTRIB" == "pip-free-threaded" ]]; then
         source $VIRTUALENV/bin/activate
     fi
 }
diff --git a/doc/developers/contributing.rst b/doc/developers/contributing.rst
@@ -518,7 +518,7 @@ Commit Message Marker  Action Taken by CI
 [cd build cirrus]      CD is run only for Cirrus CI
 [lint skip]            Azure pipeline skips linting
 [scipy-dev]            Build & test with our dependencies (numpy, scipy, etc.) development builds
-[nogil]                Build & test with the nogil experimental branches of CPython, Cython, NumPy, SciPy, ...
+[free-threaded]        Build & test with CPython 3.13 free-threaded
 [pyodide]              Build & test with Pyodide
 [azure parallel]       Run Azure CI jobs in parallel
 [cirrus arm]           Run Cirrus CI ARM test