Skip to content

Commit 2f09bbb

Browse files
alfaro96ogrisel
alfaro96
and
ogrisel
authored
MNT Use a minimal Windows Docker container to check the wheels [cd build] (#18802)
Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>
1 parent 4bc4d23 commit 2f09bbb

File tree

12 files changed

+296
-89
lines changed

12 files changed

+296
-89
lines changed

‎.github/workflows/wheels.yml

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,23 +78,19 @@ jobs:
7878

7979
- name: Build and test wheels
8080
env:
81-
# Set the directory where the wheel is unpacked
81+
CONFTEST_PATH: ${{ github.workspace }}/conftest.py
82+
CONFTEST_NAME: conftest.py
8283
CIBW_ENVIRONMENT: WHEEL_DIRNAME=scikit_learn-$SCIKIT_LEARN_VERSION
8384
OMP_NUM_THREADS=2
8485
OPENBLAS_NUM_THREADS=2
8586
SKLEARN_SKIP_NETWORK_TESTS=1
8687
SKLEARN_BUILD_PARALLEL=3
8788
CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
89+
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
90+
CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
8891
CIBW_TEST_REQUIRES: pytest pandas threadpoolctl
89-
# Test that there are no links to system libraries
90-
CIBW_TEST_COMMAND: pytest --pyargs sklearn &&
91-
python -m threadpoolctl -i sklearn
92-
# By default, the Windows wheels are not repaired.
93-
# In this case, we need to vendor the vcomp140.dll
94-
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: wheel unpack {wheel} &&
95-
python build_tools/github/vendor_vcomp140.py %WHEEL_DIRNAME% &&
96-
wheel pack %WHEEL_DIRNAME% -d {dest_dir} &&
97-
rmdir /s /q %WHEEL_DIRNAME%
92+
CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
93+
CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
9894

9995
run: bash build_tools/github/build_wheels.sh
10096

@@ -117,8 +113,17 @@ jobs:
117113
- name: Setup Python
118114
uses: actions/setup-python@v2
119115

120-
- name: Build and test source distribution
116+
- name: Build source distribution
121117
run: bash build_tools/github/build_source.sh
118+
env:
119+
SKLEARN_BUILD_PARALLEL: 3
120+
121+
- name: Test source distribution
122+
run: bash build_tools/github/test_source.sh
123+
env:
124+
OMP_NUM_THREADS: 2
125+
OPENBLAS_NUM_THREADS: 2
126+
SKLEARN_SKIP_NETWORK_TESTS: 1
122127

123128
- name: Store artifacts
124129
uses: actions/upload-artifact@v2
@@ -130,7 +135,7 @@ jobs:
130135
name: Upload to Anaconda
131136
runs-on: ubuntu-latest
132137
needs: [build_wheels, build_sdist]
133-
# The artifacts are not be uploaded on PRs
138+
# The artifacts cannot be uploaded on PRs
134139
if: github.event_name != 'pull_request'
135140

136141
steps:

‎build_tools/github/Windows

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Get the Python version of the base image from a build argument
2+
ARG PYTHON_VERSION
3+
FROM winamd64/python:$PYTHON_VERSION-windowsservercore
4+
5+
ARG WHEEL_NAME
6+
ARG CONFTEST_NAME
7+
ARG CIBW_TEST_REQUIRES
8+
9+
# Copy and install the Windows wheel
10+
COPY $WHEEL_NAME $WHEEL_NAME
11+
COPY $CONFTEST_NAME $CONFTEST_NAME
12+
RUN pip install $env:WHEEL_NAME
13+
14+
# Install the testing dependencies
15+
RUN pip install $env:CIBW_TEST_REQUIRES.split(" ")

‎build_tools/github/build_minimal_windows_image.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
PYTHON_VERSION=$1
7+
BITNESS=$2
8+
9+
if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then
10+
# Python 3.6 and 32-bit architectures are not supported
11+
# by the official Docker images: Tests will just be run
12+
# on the host (instead of the minimal Docker container).
13+
exit 0
14+
fi
15+
16+
TEMP_FOLDER="$HOME/AppData/Local/Temp"
17+
WHEEL_PATH=$(ls -d $TEMP_FOLDER/*/repaired_wheel/*)
18+
WHEEL_NAME=$(basename $WHEEL_PATH)
19+
20+
cp $WHEEL_PATH $WHEEL_NAME
21+
22+
# Dot the Python version for identyfing the base Docker image
23+
PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2})
24+
25+
# Build a minimal Windows Docker image for testing the wheels
26+
docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \
27+
--build-arg WHEEL_NAME=$WHEEL_NAME \
28+
--build-arg CONFTEST_NAME=$CONFTEST_NAME \
29+
--build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \
30+
-f build_tools/github/Windows \
31+
-t scikit-learn/minimal-windows .

‎build_tools/github/build_source.sh

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33
set -e
44
set -x
55

6+
# Move up two levels to create the virtual
7+
# environment outside of the source folder
8+
cd ../../
9+
10+
python -m venv build_env
11+
source build_env/bin/activate
12+
613
python -m pip install numpy scipy cython
714
python -m pip install twine
8-
python -m pip install pytest pandas
915

16+
cd scikit-learn/scikit-learn
1017
python setup.py sdist
11-
python -m pip install dist/*.tar.gz
12-
python setup.py build_ext -i
13-
14-
pytest --pyargs sklearn
1518

1619
# Check whether the source distribution will render correctly
1720
twine check dist/*.tar.gz

‎build_tools/github/build_wheels.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -e
44
set -x
55

66
# OpenMP is not present on macOS by default
7-
if [ "$RUNNER_OS" == "macOS" ]; then
7+
if [[ "$RUNNER_OS" == "macOS" ]]; then
88
brew install libomp
99
export CC=/usr/bin/clang
1010
export CXX=/usr/bin/clang++

‎build_tools/github/repair_windows_wheels.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
WHEEL=$1
7+
DEST_DIR=$2
8+
BITNESS=$3
9+
10+
# By default, the Windows wheels are not repaired.
11+
# In this case, we need to vendor VCRUNTIME140.dll
12+
wheel unpack "$WHEEL"
13+
python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS"
14+
wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR"
15+
rm -rf "$WHEEL_DIRNAME"

‎build_tools/github/test_source.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
cd ../../
7+
8+
python -m venv test_env
9+
source test_env/bin/activate
10+
11+
python -m pip install scikit-learn/scikit-learn/dist/*.tar.gz
12+
python -m pip install pytest pandas
13+
14+
# Run the tests on the installed source distribution
15+
mkdir tmp_for_test
16+
cp scikit-learn/scikit-learn/conftest.py tmp_for_test
17+
cd tmp_for_test
18+
19+
pytest --pyargs sklearn

‎build_tools/github/test_wheels.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
if [[ "$OSTYPE" != "linux-gnu" ]]; then
7+
# The Linux test environment is run in a Docker container and
8+
# it is not possible to copy the test configuration file (yet)
9+
cp $CONFTEST_PATH $CONFTEST_NAME
10+
fi
11+
12+
pytest --pyargs sklearn
13+
14+
# Test that there are no links to system libraries
15+
python -m threadpoolctl -i sklearn

‎build_tools/github/test_windows_wheels.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
PYTHON_VERSION=$1
7+
BITNESS=$2
8+
9+
if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then
10+
# For Python 3.6 and 32-bit architecture use the regular
11+
# test command (outside of the minimal Docker container)
12+
cp $CONFTEST_PATH $CONFTEST_NAME
13+
pytest --pyargs sklearn
14+
python -m threadpoolctl -i sklearn
15+
else
16+
docker container run -e SKLEARN_SKIP_NETWORK_TESTS=1 \
17+
-e OMP_NUM_THREADS=2 \
18+
-e OPENBLAS_NUM_THREADS=2 \
19+
--rm scikit-learn/minimal-windows \
20+
powershell -Command "pytest --pyargs sklearn"
21+
22+
docker container run --rm scikit-learn/minimal-windows \
23+
powershell -Command "python -m threadpoolctl -i sklearn"
24+
fi

‎build_tools/github/vendor.py

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll.
2+
3+
Note that vcruntime140_1.dll is only required (and available)
4+
for 64-bit architectures.
5+
"""
6+
7+
8+
import os
9+
import os.path as op
10+
import shutil
11+
import sys
12+
import textwrap
13+
14+
15+
TARGET_FOLDER = op.join("sklearn", ".libs")
16+
DISTRIBUTOR_INIT = op.join("sklearn", "_distributor_init.py")
17+
VCOMP140_SRC_PATH = "C:\\Windows\\System32\\vcomp140.dll"
18+
VCRUNTIME140_SRC_PATH = "C:\\Windows\\System32\\vcruntime140.dll"
19+
VCRUNTIME140_1_SRC_PATH = "C:\\Windows\\System32\\vcruntime140_1.dll"
20+
21+
22+
def make_distributor_init_32_bits(distributor_init,
23+
vcomp140_dll_filename,
24+
vcruntime140_dll_filename):
25+
"""Create a _distributor_init.py file for 32-bit architectures.
26+
27+
This file is imported first when importing the sklearn package
28+
so as to pre-load the vendored vcomp140.dll and vcruntime140.dll.
29+
"""
30+
with open(distributor_init, "wt") as f:
31+
f.write(textwrap.dedent("""
32+
'''Helper to preload vcomp140.dll and vcruntime140.dll to
33+
prevent "not found" errors.
34+
35+
Once vcomp140.dll and vcruntime140.dll are preloaded, the
36+
namespace is made available to any subsequent vcomp140.dll
37+
and vcruntime140.dll. This is created as part of the scripts
38+
that build the wheel.
39+
'''
40+
41+
42+
import os
43+
import os.path as op
44+
from ctypes import WinDLL
45+
46+
47+
if os.name == "nt":
48+
# Load vcomp140.dll and vcruntime140.dll
49+
libs_path = op.join(op.dirname(__file__), ".libs")
50+
vcomp140_dll_filename = op.join(libs_path, "{0}")
51+
vcruntime140_dll_filename = op.join(libs_path, "{1}")
52+
WinDLL(op.abspath(vcomp140_dll_filename))
53+
WinDLL(op.abspath(vcruntime140_dll_filename))
54+
""".format(vcomp140_dll_filename, vcruntime140_dll_filename)))
55+
56+
57+
def make_distributor_init_64_bits(distributor_init,
58+
vcomp140_dll_filename,
59+
vcruntime140_dll_filename,
60+
vcruntime140_1_dll_filename):
61+
"""Create a _distributor_init.py file for 64-bit architectures.
62+
63+
This file is imported first when importing the sklearn package
64+
so as to pre-load the vendored vcomp140.dll, vcruntime140.dll
65+
and vcruntime140_1.dll.
66+
"""
67+
with open(distributor_init, "wt") as f:
68+
f.write(textwrap.dedent("""
69+
'''Helper to preload vcomp140.dll, vcruntime140.dll and
70+
vcruntime140_1.dll to prevent "not found" errors.
71+
72+
Once vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll are
73+
preloaded, the namespace is made available to any subsequent
74+
vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll. This is
75+
created as part of the scripts that build the wheel.
76+
'''
77+
78+
79+
import os
80+
import os.path as op
81+
from ctypes import WinDLL
82+
83+
84+
if os.name == "nt":
85+
# Load vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll
86+
libs_path = op.join(op.dirname(__file__), ".libs")
87+
vcomp140_dll_filename = op.join(libs_path, "{0}")
88+
vcruntime140_dll_filename = op.join(libs_path, "{1}")
89+
vcruntime140_1_dll_filename = op.join(libs_path, "{2}")
90+
WinDLL(op.abspath(vcomp140_dll_filename))
91+
WinDLL(op.abspath(vcruntime140_dll_filename))
92+
WinDLL(op.abspath(vcruntime140_1_dll_filename))
93+
""".format(vcomp140_dll_filename,
94+
vcruntime140_dll_filename,
95+
vcruntime140_1_dll_filename)))
96+
97+
98+
def main(wheel_dirname, bitness):
99+
"""Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll."""
100+
if not op.exists(VCOMP140_SRC_PATH):
101+
raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.")
102+
103+
if not op.exists(VCRUNTIME140_SRC_PATH):
104+
raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.")
105+
106+
if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64":
107+
raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.")
108+
109+
if not op.isdir(wheel_dirname):
110+
raise RuntimeError(f"Could not find {wheel_dirname} file.")
111+
112+
vcomp140_dll_filename = op.basename(VCOMP140_SRC_PATH)
113+
vcruntime140_dll_filename = op.basename(VCRUNTIME140_SRC_PATH)
114+
vcruntime140_1_dll_filename = op.basename(VCRUNTIME140_1_SRC_PATH)
115+
116+
target_folder = op.join(wheel_dirname, TARGET_FOLDER)
117+
distributor_init = op.join(wheel_dirname, DISTRIBUTOR_INIT)
118+
119+
# Create the "sklearn/.libs" subfolder
120+
if not op.exists(target_folder):
121+
os.mkdir(target_folder)
122+
123+
print(f"Copying {VCOMP140_SRC_PATH} to {target_folder}.")
124+
shutil.copy2(VCOMP140_SRC_PATH, target_folder)
125+
126+
print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.")
127+
shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder)
128+
129+
if bitness == "64":
130+
print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
131+
shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
132+
133+
# Generate the _distributor_init file in the source tree
134+
print("Generating the '_distributor_init.py' file.")
135+
if bitness == "32":
136+
make_distributor_init_32_bits(distributor_init,
137+
vcomp140_dll_filename,
138+
vcruntime140_dll_filename)
139+
else:
140+
make_distributor_init_64_bits(distributor_init,
141+
vcomp140_dll_filename,
142+
vcruntime140_dll_filename,
143+
vcruntime140_1_dll_filename)
144+
145+
146+
if __name__ == "__main__":
147+
_, wheel_file, bitness = sys.argv
148+
main(wheel_file, bitness)

0 commit comments

Comments
 (0)