Skip to content

Commit 96de4e2

Browse files
jjerphanglemaitreogriselcmarmothomasjpfan
committed
CI Adapt Circle CI script for ARM (#21174)
Co-authored-by: Guillaume Lemaitre <g.lemaitre58@gmail.com> Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org> Co-authored-by: Chiara Marmo <cmarmo@users.noreply.github.com> Co-authored-by: Thomas J. Fan <thomasjpfan@gmail.com>
1 parent fe2bee1 commit 96de4e2

File tree

2 files changed

+62
-32
lines changed

2 files changed

+62
-32
lines changed

‎.circleci/config.yml

+9-7
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,12 @@ jobs:
114114
image: ubuntu-2004:202101-01
115115
resource_class: arm.medium
116116
environment:
117+
# Use the latest supported version of python
118+
- PYTHON_VERSION: '3.9'
117119
- OMP_NUM_THREADS: 2
118120
- OPENBLAS_NUM_THREADS: 2
121+
- NUMPY_VERSION: 'latest'
122+
- SCIPY_VERSION: 'latest'
119123
- CYTHON_VERSION: 'latest'
120124
- JOBLIB_VERSION: 'latest'
121125
- THREADPOOLCTL_VERSION: 'latest'
@@ -126,18 +130,16 @@ jobs:
126130
- checkout
127131
- run: ./build_tools/circle/checkout_merge_commit.sh
128132
- restore_cache:
129-
key: v1-datasets-{{ .Branch }}
133+
key: linux-arm64-{{ .Branch }}
130134
- run: ./build_tools/circle/build_test_arm.sh
131135
- save_cache:
132-
key: doc-ccache-{{ .Branch }}-{{ .BuildNum }}
136+
key: linux-arm64-{{ .Branch }}
133137
paths:
134-
- ~/.ccache
138+
- ~/.cache/ccache
135139
- ~/.cache/pip
136-
- save_cache:
137-
key: v1-datasets-{{ .Branch }}
138-
paths:
139140
- ~/scikit_learn_data
140-
141+
# The source build folder.
142+
- ~/project/build
141143
deploy:
142144
docker:
143145
- image: circleci/python:3.7

‎build_tools/circle/build_test_arm.sh

+53-25
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ set -e
44
set -x
55

66
UNAMESTR=`uname`
7+
N_CORES=`nproc --all`
8+
79

810
setup_ccache() {
911
echo "Setting up ccache"
@@ -12,49 +14,75 @@ setup_ccache() {
1214
for name in gcc g++ cc c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++; do
1315
ln -s $(which ccache) "/tmp/ccache/${name}"
1416
done
15-
export PATH="/tmp/ccache/:${PATH}"
16-
ccache -M 256M
17+
export PATH="/tmp/ccache:${PATH}"
18+
# Unset ccache limits
19+
ccache -F 0
20+
ccache -M 0
1721
}
1822

1923
# imports get_dep
2024
source build_tools/shared.sh
2125

2226
sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
2327
sudo apt-get update
24-
sudo apt-get install python3-virtualenv ccache
25-
python3 -m virtualenv --system-site-packages --python=python3 testenv
26-
source testenv/bin/activate
27-
pip install --upgrade pip
28+
29+
# Setup conda environment
30+
MINICONDA_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh"
31+
32+
# Install Mambaforge
33+
wget $MINICONDA_URL -O mambaforge.sh
34+
MINICONDA_PATH=$HOME/miniconda
35+
chmod +x mambaforge.sh && ./mambaforge.sh -b -p $MINICONDA_PATH
36+
export PATH=$MINICONDA_PATH/bin:$PATH
37+
mamba init --all --verbose
38+
mamba update --yes conda
39+
40+
# Create environment and install dependencies
41+
mamba create -n testenv --yes $(get_dep python $PYTHON_VERSION)
42+
source activate testenv
43+
44+
# Use the latest by default
45+
mamba install --verbose -y ccache \
46+
pip \
47+
$(get_dep numpy $NUMPY_VERSION) \
48+
$(get_dep scipy $SCIPY_VERSION) \
49+
$(get_dep cython $CYTHON_VERSION) \
50+
$(get_dep joblib $JOBLIB_VERSION) \
51+
$(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \
52+
$(get_dep pytest $PYTEST_VERSION) \
53+
$(get_dep pytest-xdist $PYTEST_XDIST_VERSION)
2854
setup_ccache
29-
python -m pip install $(get_dep cython $CYTHON_VERSION) \
30-
$(get_dep joblib $JOBLIB_VERSION)
31-
python -m pip install $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \
32-
$(get_dep pytest $PYTEST_VERSION) \
33-
$(get_dep pytest-xdist $PYTEST_XDIST_VERSION)
3455

3556
if [[ "$COVERAGE" == "true" ]]; then
36-
python -m pip install codecov pytest-cov
37-
fi
38-
39-
if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then
40-
python -m pip install pytest-xdist
57+
mamba install --verbose -y codecov pytest-cov
4158
fi
4259

4360
if [[ "$TEST_DOCSTRINGS" == "true" ]]; then
4461
# numpydoc requires sphinx
45-
python -m pip install sphinx
46-
python -m pip install numpydoc
62+
mamba install --verbose -y sphinx
63+
mamba install --verbose -y numpydoc
4764
fi
4865

4966
python --version
5067

51-
# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
52-
# workers with 2 cores when building the compiled extensions of scikit-learn.
53-
export SKLEARN_BUILD_PARALLEL=3
68+
# Set parallelism to $N_CORES + 1 to overlap IO bound tasks with CPU bound tasks on CI
69+
# workers with $N_CORES cores when building the compiled extensions of scikit-learn.
70+
export SKLEARN_BUILD_PARALLEL=$(($N_CORES + 1))
71+
72+
# Disable the build isolation and build in the tree so that the same folder can be
73+
# cached between CI runs.
74+
# TODO: remove the '--use-feature' flag when made obsolete in pip 21.3.
75+
pip install --verbose --no-build-isolation --use-feature=in-tree-build .
76+
77+
# Report cache usage
78+
ccache -s --verbose
79+
80+
mamba list
5481

55-
python -m pip list
56-
pip install --verbose --editable .
57-
ccache -s
82+
# Changing directory not to have module resolution use scikit-learn source
83+
# directory but to the installed package.
84+
cd /tmp
5885
python -c "import sklearn; sklearn.show_versions()"
5986
python -m threadpoolctl --import sklearn
60-
python -m pytest sklearn
87+
# Test using as many workers as available cores
88+
pytest --pyargs -n $N_CORES sklearn

0 commit comments

Comments
 (0)