From 3f13773a54051dcdd7f13b66251d82a37ef0afa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 13:12:59 +0200 Subject: [PATCH 01/90] [DOC] Move documentation builder to actions. --- .github/workflows/docs.yml | 38 ++++++++++ build_tools/circle/build_doc.sh | 127 ++++++-------------------------- build_tools/circle/push_doc.sh | 17 +---- 3 files changed, 60 insertions(+), 122 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000000..9c3e593e1a85c --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,38 @@ +# Workflow to build and publish the documentation +name: Documentation builder + +on: + schedule: + # Nightly build at 3:42 A.M. + - cron: "42 3 */1 * *" + push: + branches: + - main + # Release branches + - "[0-9]+.[0-9]+.X" + pull_request: + branches: + - main + - "[0-9]+.[0-9]+.X" + # Manual run + workflow_dispatch: + +jobs: + # + build: + runs-on: ubuntu-latest + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build documentation + run: build_tools/.github/build_docs.sh + + - name: Upload documentation + uses: actions/upload-artifact@v2 + with: + name: docs + path: doc/_build/html/stable diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 8facdc59a6c56..85d6ce61274e7 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -18,15 +18,15 @@ set -e # behavior is to quick build the documentation. get_build_type() { - if [ -z "$CIRCLE_SHA1" ] + if [ -z "$GITHUB_SHA" ] then - echo SKIP: undefined CIRCLE_SHA1 + echo SKIP: undefined GITHUB_SHA return fi - commit_msg=$(git log --format=%B -n 1 $CIRCLE_SHA1) + commit_msg=$(git log --format=%B -n 1 $GITHUB_SHA) if [ -z "$commit_msg" ] then - echo QUICK BUILD: failed to inspect commit $CIRCLE_SHA1 + echo QUICK BUILD: failed to inspect commit $GITHUB_SHA return fi if [[ "$commit_msg" =~ \[doc\ skip\] ]] @@ -44,12 +44,12 @@ get_build_type() { echo BUILD: [doc build] marker found return fi - if [ -z "$CI_PULL_REQUEST" ] + if [[ "$GITHUB_EVENT_NAME" != pull_request ]] then echo BUILD: not a pull request return fi - git_range="origin/main...$CIRCLE_SHA1" + git_range="origin/main...$GITHUB_SHA" git fetch origin main >&2 || (echo QUICK BUILD: failed to get changed filenames for $git_range; return) filenames=$(git diff --name-only $git_range) if [ -z "$filenames" ] @@ -114,7 +114,7 @@ then exit 0 fi -if [[ "$CIRCLE_BRANCH" =~ ^main$|^[0-9]+\.[0-9]+\.X$ && -z "$CI_PULL_REQUEST" ]] +if [[ "$GITHUB_REF" =~ ^main$|^[0-9]+\.[0-9]+\.X$ && "$GITHUB_EVENT_NAME" != pull_request ]] then # ZIP linked into HTML make_args=dist @@ -138,49 +138,21 @@ sudo -E apt-get -yq update --allow-releaseinfo-change sudo -E apt-get -yq --no-install-suggests --no-install-recommends \ install dvipng gsfonts ccache zip optipng -# deactivate circleci virtualenv and setup a miniconda env instead -if [[ `type -t deactivate` ]]; then - deactivate -fi - -MINICONDA_PATH=$HOME/miniconda -# Install dependencies with miniconda -wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - -O miniconda.sh -chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH -export PATH="/usr/lib/ccache:$MINICONDA_PATH/bin:$PATH" - -ccache -M 512M -export CCACHE_COMPRESS=1 - -# Old packages coming from the 'free' conda channel have been removed but we -# are using them for our min-dependencies doc generation. See -# https://www.anaconda.com/why-we-removed-the-free-channel-in-conda-4-7/ for -# more details. -if [[ "$CIRCLE_JOB" == "doc-min-dependencies" ]]; then - conda config --set restore_free_channel true -fi - # imports get_dep source build_tools/shared.sh -# packaging won't be needed once setuptools starts shipping packaging>=17.0 -conda create -n $CONDA_ENV_NAME --yes --quiet \ - python="${PYTHON_VERSION:-*}" \ - "$(get_dep numpy $NUMPY_VERSION)" \ - "$(get_dep scipy $SCIPY_VERSION)" \ - "$(get_dep cython $CYTHON_VERSION)" \ - "$(get_dep matplotlib $MATPLOTLIB_VERSION)" \ - "$(get_dep sphinx $SPHINX_VERSION)" \ - "$(get_dep pandas $PANDAS_VERSION)" \ - joblib memory_profiler packaging seaborn pillow pytest coverage - -source activate testenv -pip install "$(get_dep scikit-image $SCIKIT_IMAGE_VERSION)" -pip install "$(get_dep sphinx-gallery $SPHINX_GALLERY_VERSION)" -pip install "$(get_dep numpydoc $NUMPYDOC_VERSION)" -pip install "$(get_dep sphinx-prompt $SPHINX_PROMPT_VERSION)" -pip install "$(get_dep sphinxext-opengraph $SPHINXEXT_OPENGRAPH_VERSION)" +pip install "$(get_dep numpy $NUMPY_VERSION)" \ + "$(get_dep scipy $SCIPY_VERSION)" \ + "$(get_dep cython $CYTHON_VERSION)" \ + "$(get_dep matplotlib $MATPLOTLIB_VERSION)" \ + "$(get_dep sphinx $SPHINX_VERSION)" \ + "$(get_dep pandas $PANDAS_VERSION)" \ + "$(get_dep scikit-image $SCIKIT_IMAGE_VERSION)" + "$(get_dep sphinx-gallery $SPHINX_GALLERY_VERSION)" + "$(get_dep numpydoc $NUMPYDOC_VERSION)" + "$(get_dep sphinx-prompt $SPHINX_PROMPT_VERSION)" + "$(get_dep sphinxext-opengraph $SPHINXEXT_OPENGRAPH_VERSION)" + joblib memory_profiler seaborn pillow pytest coverage # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI # workers with 2 cores when building the compiled extensions of scikit-learn. @@ -189,72 +161,15 @@ python setup.py develop export OMP_NUM_THREADS=1 -if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]] +if [[ "$GITHUB_SHA" =~ ^main$ && "$GITHUB_EVENT_NAME" == pull_request ]] then # List available documentation versions if on main python build_tools/circle/list_versions.py > doc/versions.rst fi # The pipefail is requested to propagate exit code -set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt +cd doc && make $make_args # Insert the version warning for deployment find _build/html/stable -name "*.html" | xargs sed -i '/<\/body>/ i \ \ ' - -cd - -set +o pipefail - -affected_doc_paths() { - files=$(git diff --name-only origin/main...$CIRCLE_SHA1) - echo "$files" | grep ^doc/.*\.rst | sed 's/^doc\/\(.*\)\.rst$/\1.html/' - echo "$files" | grep ^examples/.*.py | sed 's/^\(.*\)\.py$/auto_\1.html/' - sklearn_files=$(echo "$files" | grep '^sklearn/') - if [ -n "$sklearn_files" ] - then - grep -hlR -f<(echo "$sklearn_files" | sed 's/^/scikit-learn\/blob\/[a-z0-9]*\//') doc/_build/html/stable/modules/generated | cut -d/ -f5- - fi -} - -affected_doc_warnings() { - files=$(git diff --name-only origin/main...$CIRCLE_SHA1) - # Look for sphinx warnings only in files affected by the PR - if [ -n "$files" ] - then - for af in ${files[@]} - do - warn+=`grep WARNING ~/log.txt | grep $af` - done - fi - echo "$warn" -} - -if [ -n "$CI_PULL_REQUEST" ] -then - echo "The following documentation warnings may have been generated by PR #$CI_PULL_REQUEST:" - warnings=$(affected_doc_warnings) - if [ -z "$warnings" ] - then - warnings="/home/circleci/project/ no warnings" - fi - echo "$warnings" - - echo "The following documentation files may have been changed by PR #$CI_PULL_REQUEST:" - affected=$(affected_doc_paths) - echo "$affected" - ( - echo '

General: Home | API Reference | Examples

' - echo 'Sphinx Warnings in affected files' - ) > 'doc/_build/html/stable/_changed.html' - - if [ "$warnings" != "/home/circleci/project/ no warnings" ] - then - echo "Sphinx generated warnings when building the documentation related to files modified in this PR." - echo "Please check doc/_build/html/stable/_changed.html" - exit 1 - fi -fi diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 5b94211e4e30e..526c1dc187ec1 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -6,22 +6,7 @@ set -ex -if [ -z $CIRCLE_PROJECT_USERNAME ]; -then USERNAME="sklearn-ci"; -else USERNAME=$CIRCLE_PROJECT_USERNAME; -fi - DOC_REPO="scikit-learn.github.io" -GENERATED_DOC_DIR=$1 - -if [[ -z "$GENERATED_DOC_DIR" ]]; then - echo "Need to pass directory of the generated doc as argument" - echo "Usage: $0 " - exit 1 -fi - -# Absolute path needed because we use cd further down in this script -GENERATED_DOC_DIR=$(readlink -f $GENERATED_DOC_DIR) if [ "$CIRCLE_BRANCH" = "main" ] then @@ -57,7 +42,7 @@ then fi cp -R $GENERATED_DOC_DIR $dir git config user.email "olivier.grisel+sklearn-ci@gmail.com" -git config user.name $USERNAME +git config user.name "sklearn-ci" git config push.default matching git add -f $dir/ git commit -m "$MSG" $dir From 7d0deb29a4d55619f03d313122dc01124cdc4650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 13:20:01 +0200 Subject: [PATCH 02/90] [FIX] Fix path to documentation builder script. --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9c3e593e1a85c..8f00006b7c769 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: uses: actions/setup-python@v2 - name: Build documentation - run: build_tools/.github/build_docs.sh + run: build_tools/circle/build_doc.sh - name: Upload documentation uses: actions/upload-artifact@v2 From cade5f7d039ac5243c5028a226db94bcf5b8eede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 13:22:06 +0200 Subject: [PATCH 03/90] [FIX] Fix pip install sentence. --- build_tools/circle/build_doc.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 85d6ce61274e7..a42c0d7b9b18e 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -147,11 +147,11 @@ pip install "$(get_dep numpy $NUMPY_VERSION)" \ "$(get_dep matplotlib $MATPLOTLIB_VERSION)" \ "$(get_dep sphinx $SPHINX_VERSION)" \ "$(get_dep pandas $PANDAS_VERSION)" \ - "$(get_dep scikit-image $SCIKIT_IMAGE_VERSION)" - "$(get_dep sphinx-gallery $SPHINX_GALLERY_VERSION)" - "$(get_dep numpydoc $NUMPYDOC_VERSION)" - "$(get_dep sphinx-prompt $SPHINX_PROMPT_VERSION)" - "$(get_dep sphinxext-opengraph $SPHINXEXT_OPENGRAPH_VERSION)" + "$(get_dep scikit-image $SCIKIT_IMAGE_VERSION)" \ + "$(get_dep sphinx-gallery $SPHINX_GALLERY_VERSION)" \ + "$(get_dep numpydoc $NUMPYDOC_VERSION)" \ + "$(get_dep sphinx-prompt $SPHINX_PROMPT_VERSION)" \ + "$(get_dep sphinxext-opengraph $SPHINXEXT_OPENGRAPH_VERSION)" \ joblib memory_profiler seaborn pillow pytest coverage # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI From f7f288bb018e1f86eb21a0584a5eafaa4ac5fbb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 13:27:02 +0200 Subject: [PATCH 04/90] [FIX] Fix dependency versions. --- .circleci/config.yml | 123 +------------------------------------ .github/workflows/docs.yml | 15 +++++ 2 files changed, 16 insertions(+), 122 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e8958a2ab879..23fdd8bc71de7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,101 +1,6 @@ version: 2.1 jobs: - doc-min-dependencies: - docker: - - image: circleci/python:3.7.7-buster - environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.7 - - NUMPY_VERSION: 'min' - - SCIPY_VERSION: 'min' - - MATPLOTLIB_VERSION: 'min' - - CYTHON_VERSION: 'min' - - SCIKIT_IMAGE_VERSION: 'min' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'min' - - SPHINX_GALLERY_VERSION: 'min' - - NUMPYDOC_VERSION: 'min' - - SPHINX_PROMPT_VERSION: 'min' - - SPHINXEXT_OPENGRAPH_VERSION: 'min' - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-min-deps-ccache-{{ .Branch }} - - doc-min-deps-ccache - - run: ./build_tools/circle/build_doc.sh - - save_cache: - key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data - - store_artifacts: - path: doc/_build/html/stable - destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt - - doc: - docker: - - image: circleci/python:3.7.7-buster - environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3 - - NUMPY_VERSION: 'latest' - - SCIPY_VERSION: 'latest' - - MATPLOTLIB_VERSION: 'latest' - - CYTHON_VERSION: 'latest' - - SCIKIT_IMAGE_VERSION: 'latest' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'latest' - - SPHINX_GALLERY_VERSION: 'latest' - - NUMPYDOC_VERSION: 'latest' - - SPHINX_PROMPT_VERSION: 'latest' - - SPHINXEXT_OPENGRAPH_VERSION: 'latest' - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-ccache-{{ .Branch }} - - doc-ccache - - run: ./build_tools/circle/build_doc.sh - - save_cache: - key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data - - store_artifacts: - path: doc/_build/html/stable - destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt - # Persists generated documentation so that it can be attached and deployed - # in the 'deploy' step. - - persist_to_workspace: - root: doc/_build/html - paths: . - lint: docker: - image: circleci/python:3.7 @@ -138,37 +43,11 @@ jobs: paths: - ~/scikit_learn_data - deploy: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - # Attach documentation generated in the 'doc' step so that it can be - # deployed. - - attach_workspace: - at: doc/_build/html - - run: ls -ltrh doc/_build/html/stable - - deploy: - command: | - if [[ "${CIRCLE_BRANCH}" =~ ^main$|^[0-9]+\.[0-9]+\.X$ ]]; then - bash build_tools/circle/push_doc.sh doc/_build/html/stable - fi - workflows: version: 2 - build-doc-and-deploy: + lint: jobs: - lint - - doc: - requires: - - lint - - doc-min-dependencies: - requires: - - lint - - deploy: - requires: - - doc linux-arm64: jobs: - linux-arm64 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f00006b7c769..158302db4dbef 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,6 +30,21 @@ jobs: - name: Build documentation run: build_tools/circle/build_doc.sh + with: + env: + OMP_NUM_THREADS: 2 + MKL_NUM_THREADS: 2 + NUMPY_VERSION: latest + SCIPY_VERSION: latest + MATPLOTLIB_VERSION: latest + CYTHON_VERSION: latest + SCIKIT_IMAGE_VERSION: latest + SPHINX_VERSION: min + PANDAS_VERSION: latest + SPHINX_GALLERY_VERSION: latest + NUMPYDOC_VERSION: latest + SPHINX_PROMPT_VERSION: latest + SPHINXEXT_OPENGRAPH_VERSION: latest - name: Upload documentation uses: actions/upload-artifact@v2 From 90575649532e0831aec600dca4a5f2568ee3f88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 13:28:17 +0200 Subject: [PATCH 05/90] [FIX] Fix action syntax. --- .github/workflows/docs.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 158302db4dbef..829a2072c6a07 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,21 +30,20 @@ jobs: - name: Build documentation run: build_tools/circle/build_doc.sh - with: - env: - OMP_NUM_THREADS: 2 - MKL_NUM_THREADS: 2 - NUMPY_VERSION: latest - SCIPY_VERSION: latest - MATPLOTLIB_VERSION: latest - CYTHON_VERSION: latest - SCIKIT_IMAGE_VERSION: latest - SPHINX_VERSION: min - PANDAS_VERSION: latest - SPHINX_GALLERY_VERSION: latest - NUMPYDOC_VERSION: latest - SPHINX_PROMPT_VERSION: latest - SPHINXEXT_OPENGRAPH_VERSION: latest + env: + OMP_NUM_THREADS: 2 + MKL_NUM_THREADS: 2 + NUMPY_VERSION: latest + SCIPY_VERSION: latest + MATPLOTLIB_VERSION: latest + CYTHON_VERSION: latest + SCIKIT_IMAGE_VERSION: latest + SPHINX_VERSION: min + PANDAS_VERSION: latest + SPHINX_GALLERY_VERSION: latest + NUMPYDOC_VERSION: latest + SPHINX_PROMPT_VERSION: latest + SPHINXEXT_OPENGRAPH_VERSION: latest - name: Upload documentation uses: actions/upload-artifact@v2 From e5887b00ea3fcee05614716d0af06bb945b7e9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 17:04:51 +0200 Subject: [PATCH 06/90] [CI] Add push step. --- .github/workflows/docs.yml | 25 ++++++++++++++++++- build_tools/circle/push_doc.sh | 45 ++++++++++++++++------------------ 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 829a2072c6a07..fd08bb43a1991 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ on: workflow_dispatch: jobs: - # + # Build the documentation build: runs-on: ubuntu-latest steps: @@ -50,3 +50,26 @@ jobs: with: name: docs path: doc/_build/html/stable + + # Push the documentation + push: + runs-on: ubuntu-latest + steps: + - name: Download documentation + uses: actions/download-artifact@v2 + with: + name: docs + + - name: Checkout scikit-learn + uses: actions/checkout@v1 + with: + path: scikit-learn + + - name: Checkout scikit-learn.github.io + uses: actions/checkout@v2 + with: + path: scikit-learn.github.io + ssh-key: ${{ secrets.DOC_BUILDER_KEY }} + + - name: Push documentation + run: $GITHUB_WORKSPACE/scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 526c1dc187ec1..c31a49b7a088c 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -1,50 +1,47 @@ #!/bin/bash -# This script is meant to be called in the "deploy" step defined in -# circle.yml. See https://circleci.com/docs/ for more details. -# The behavior of the script is controlled by environment variable defined -# in the circle.yml in the top level folder of the project. + +# The behavior of the script is controlled by environment variable +# defined in the docs.yml in the actions level folder of the project. set -ex DOC_REPO="scikit-learn.github.io" -if [ "$CIRCLE_BRANCH" = "main" ] +if [ "$GITHUB_REF" =~ "main" ] then - dir=dev + DIR=dev else # Strip off .X - dir="${CIRCLE_BRANCH::-2}" + DIR="${GITHUB_REF::-2}" fi -MSG="Pushing the docs to $dir/ for branch: $CIRCLE_BRANCH, commit $CIRCLE_SHA1" - -cd $HOME -if [ ! -d $DOC_REPO ]; -then git clone --depth 1 --no-checkout "git@github.com:scikit-learn/"$DOC_REPO".git"; -fi -cd $DOC_REPO +MSG="Pushing the docs to $DIR/ for branch: $GITHUB_REF, commit $GITHUB_SHA" # check if it's a new branch +echo $DIR > .git/info/sparse-checkout -echo $dir > .git/info/sparse-checkout -if ! git show HEAD:$dir >/dev/null +if ! git show HEAD:$DIR >/dev/null then # directory does not exist. Need to make it so sparse checkout works - mkdir $dir - touch $dir/index.html - git add $dir + mkdir $DIR + touch $DIR/index.html + git add $DIR fi git checkout main git reset --hard origin/main -if [ -d $dir ] +if [ -d $DIR ] then - git rm -rf $dir/ && rm -rf $dir/ + git rm -rf $DIR/ && rm -rf $DIR/ fi -cp -R $GENERATED_DOC_DIR $dir + +cp -R $GITHUB_WORKSPACE/docs $DIR + git config user.email "olivier.grisel+sklearn-ci@gmail.com" git config user.name "sklearn-ci" git config push.default matching -git add -f $dir/ -git commit -m "$MSG" $dir + +git add -f $DIR/ +git commit -m "$MSG" $DIR git push + echo $MSG From 1899ea9ba06152cfef501f5dac187afdc0f0f82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 17:06:27 +0200 Subject: [PATCH 07/90] [FIX] Add new step. --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fd08bb43a1991..0590ca263822c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -54,6 +54,7 @@ jobs: # Push the documentation push: runs-on: ubuntu-latest + needs: [build] steps: - name: Download documentation uses: actions/download-artifact@v2 From 1c0546c1ab174aa57bda1ef9b66e3d14c3d5c2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 17:26:29 +0200 Subject: [PATCH 08/90] [FIX] Fix path to shell script. --- .github/workflows/docs.yml | 12 ++++++------ build_tools/circle/push_doc.sh | 5 ++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0590ca263822c..9a0ba06db189d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -61,16 +61,16 @@ jobs: with: name: docs - - name: Checkout scikit-learn - uses: actions/checkout@v1 - with: - path: scikit-learn - - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 with: path: scikit-learn.github.io ssh-key: ${{ secrets.DOC_BUILDER_KEY }} + - name: Checkout scikit-learn + uses: actions/checkout@v1 + with: + path: scikit-learn + - name: Push documentation - run: $GITHUB_WORKSPACE/scikit-learn/build_tools/circle/push_doc.sh + run: ../scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index c31a49b7a088c..def4e7311fb11 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -34,7 +34,10 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi -cp -R $GITHUB_WORKSPACE/docs $DIR +cp -R ../docs $DIR + +ls -l ../docs +ls -l $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" git config user.name "sklearn-ci" From a79cd1e7a0997e88a4599645332cd391842d0e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 17:50:09 +0200 Subject: [PATCH 09/90] [FIX] Fix commit SHA. --- .github/workflows/docs.yml | 7 ++++--- build_tools/circle/push_doc.sh | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9a0ba06db189d..1aedc184d3481 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Setup Python uses: actions/setup-python@v2 @@ -64,11 +64,12 @@ jobs: - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 with: - path: scikit-learn.github.io + ref: ${{ github.event.pull_request.head.sha }} ssh-key: ${{ secrets.DOC_BUILDER_KEY }} + path: scikit-learn.github.io - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: path: scikit-learn diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index def4e7311fb11..8821b5ad780e6 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -6,16 +6,17 @@ set -ex DOC_REPO="scikit-learn.github.io" +BRANCH=$(basename "$GITHUB_REF") if [ "$GITHUB_REF" =~ "main" ] then DIR=dev else # Strip off .X - DIR="${GITHUB_REF::-2}" + DIR="${BRANCH::-2}" fi -MSG="Pushing the docs to $DIR/ for branch: $GITHUB_REF, commit $GITHUB_SHA" +MSG="Pushing the docs to $DIR/ for branch: $BRANCH, commit $GITHUB_SHA" # check if it's a new branch echo $DIR > .git/info/sparse-checkout From 0279c08f30d79e892db51505a9b942304b74ab43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:05:39 +0200 Subject: [PATCH 10/90] [FIX] Fix repository to fetch. --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1aedc184d3481..ad28f27419921 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -64,6 +64,7 @@ jobs: - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 with: + repository: scikit-learn/scikit-learn.github.io ref: ${{ github.event.pull_request.head.sha }} ssh-key: ${{ secrets.DOC_BUILDER_KEY }} path: scikit-learn.github.io From 21fbec0d660483293c313a53f6f4acd9c54a1164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:25:31 +0200 Subject: [PATCH 11/90] [FIX] Fix ref to fetch. --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ad28f27419921..dc38a362ff2ac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,13 +65,13 @@ jobs: uses: actions/checkout@v2 with: repository: scikit-learn/scikit-learn.github.io - ref: ${{ github.event.pull_request.head.sha }} ssh-key: ${{ secrets.DOC_BUILDER_KEY }} path: scikit-learn.github.io - name: Checkout scikit-learn uses: actions/checkout@v2 with: + ref: ${{ github.event.pull_request.head.sha }} path: scikit-learn - name: Push documentation From 6e2102079eb879beebf75747d53afb951ce3a768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:40:25 +0200 Subject: [PATCH 12/90] [DBG] Debugging paths. --- .github/workflows/docs.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc38a362ff2ac..77ebb6e00128a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -54,7 +54,7 @@ jobs: # Push the documentation push: runs-on: ubuntu-latest - needs: [build] + # needs: [build] steps: - name: Download documentation uses: actions/download-artifact@v2 @@ -75,4 +75,7 @@ jobs: path: scikit-learn - name: Push documentation - run: ../scikit-learn/build_tools/circle/push_doc.sh + # run: ../scikit-learn/build_tools/circle/push_doc.sh + run: | + ls -l + ls -l .. From 690fee6cf73ca3fcc16ec1ab11d4807901549c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:41:25 +0200 Subject: [PATCH 13/90] [DBG] Debugging paths. --- .github/workflows/docs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 77ebb6e00128a..48285252df990 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -56,10 +56,10 @@ jobs: runs-on: ubuntu-latest # needs: [build] steps: - - name: Download documentation - uses: actions/download-artifact@v2 - with: - name: docs + # - name: Download documentation + # uses: actions/download-artifact@v2 + # with: + # name: docs - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 From 977bbeb76b6a5715145a0c20d8f99e3a111e54a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:47:48 +0200 Subject: [PATCH 14/90] [DBG] Debugging paths. --- .github/workflows/docs.yml | 17 +++++++---------- build_tools/circle/push_doc.sh | 9 ++++++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 48285252df990..af73f5fd3175a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -54,18 +54,18 @@ jobs: # Push the documentation push: runs-on: ubuntu-latest - # needs: [build] + needs: [build] steps: - # - name: Download documentation - # uses: actions/download-artifact@v2 - # with: - # name: docs + - name: Download documentation + uses: actions/download-artifact@v2 + with: + name: docs - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 with: repository: scikit-learn/scikit-learn.github.io - ssh-key: ${{ secrets.DOC_BUILDER_KEY }} + ssh-key: ${{ secrets.SSK_KEY }} path: scikit-learn.github.io - name: Checkout scikit-learn @@ -75,7 +75,4 @@ jobs: path: scikit-learn - name: Push documentation - # run: ../scikit-learn/build_tools/circle/push_doc.sh - run: | - ls -l - ls -l .. + run: scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 8821b5ad780e6..e6f775da710a0 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -5,6 +5,8 @@ set -ex +cd scikit-learn.github.io + DOC_REPO="scikit-learn.github.io" BRANCH=$(basename "$GITHUB_REF") @@ -35,10 +37,11 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi -cp -R ../docs $DIR +ls -l . +ls -l .. +ls -l $GITHUB_WORKSPACE/docs -ls -l ../docs -ls -l $DIR +cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" git config user.name "sklearn-ci" From 8fb6600a206ebac4d1c3f637ed49adc3d763c958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:48:34 +0200 Subject: [PATCH 15/90] [FIX] Fix path. --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index af73f5fd3175a..c321bab26709b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,7 @@ jobs: - name: Download documentation uses: actions/download-artifact@v2 with: - name: docs + name: docs - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 From 9d39ea434a6df281b5d582fc0490c676141c9250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 18:49:25 +0200 Subject: [PATCH 16/90] [FIX] Fix action. --- .github/workflows/docs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c321bab26709b..3e15f032ac649 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -56,10 +56,10 @@ jobs: runs-on: ubuntu-latest needs: [build] steps: - - name: Download documentation - uses: actions/download-artifact@v2 - with: - name: docs + - name: Download documentation + uses: actions/download-artifact@v2 + with: + name: docs - name: Checkout scikit-learn.github.io uses: actions/checkout@v2 From f5c6c6348a5a7b62924a4e61ba827ca9d71b73ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 19:19:47 +0200 Subject: [PATCH 17/90] [FIX] Fix paths. --- .github/workflows/docs.yml | 16 +++++++--------- build_tools/circle/push_doc.sh | 17 +++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e15f032ac649..e8c709129adba 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn - uses: actions/checkout@v2 + uses: actions/checkout@v1 - name: Setup Python uses: actions/setup-python@v2 @@ -61,18 +61,16 @@ jobs: with: name: docs + - name: Checkout scikit-learn + uses: actions/checkout@v1 + with: + path: scikit-learn + - name: Checkout scikit-learn.github.io - uses: actions/checkout@v2 + uses: actions/checkout@v1 with: repository: scikit-learn/scikit-learn.github.io - ssh-key: ${{ secrets.SSK_KEY }} path: scikit-learn.github.io - - name: Checkout scikit-learn - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - path: scikit-learn - - name: Push documentation run: scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index e6f775da710a0..253ae352f0a8e 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -8,17 +8,22 @@ set -ex cd scikit-learn.github.io DOC_REPO="scikit-learn.github.io" -BRANCH=$(basename "$GITHUB_REF") -if [ "$GITHUB_REF" =~ "main" ] +if [ -z "$GITHUB_BASE_REF" ]; then + REF="$GITHUB_REF" +else + REF="$GITHUB_BASE_REF" +fi + +if [ "$REF" = "main" ] then DIR=dev else # Strip off .X - DIR="${BRANCH::-2}" + DIR="${REF::-2}" fi -MSG="Pushing the docs to $DIR/ for branch: $BRANCH, commit $GITHUB_SHA" +MSG="Pushing the docs to $DIR/ for branch: $REF, commit $GITHUB_SHA" # check if it's a new branch echo $DIR > .git/info/sparse-checkout @@ -37,10 +42,6 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi -ls -l . -ls -l .. -ls -l $GITHUB_WORKSPACE/docs - cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" From 5efa80580ff976a0690aa2a3c9bd78a68120cf06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 20:58:08 +0200 Subject: [PATCH 18/90] [FIX] Fix paths. --- .github/workflows/docs.yml | 6 +++--- build_tools/circle/push_doc.sh | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e8c709129adba..3d22874946b1b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Setup Python uses: actions/setup-python@v2 @@ -62,12 +62,12 @@ jobs: name: docs - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: path: scikit-learn - name: Checkout scikit-learn.github.io - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: repository: scikit-learn/scikit-learn.github.io path: scikit-learn.github.io diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 253ae352f0a8e..17c999a15dbdd 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -15,8 +15,7 @@ else REF="$GITHUB_BASE_REF" fi -if [ "$REF" = "main" ] -then +if [ "$REF" =~ "main" ]; then DIR=dev else # Strip off .X From aa84276d4e2527f37af08683394eda49f107952e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 21:16:40 +0200 Subject: [PATCH 19/90] [FIX] Fix condition. --- build_tools/circle/push_doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 17c999a15dbdd..a03426769c3d8 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -9,13 +9,13 @@ cd scikit-learn.github.io DOC_REPO="scikit-learn.github.io" -if [ -z "$GITHUB_BASE_REF" ]; then +if [[ -z "$GITHUB_BASE_REF" ]]; then REF="$GITHUB_REF" else REF="$GITHUB_BASE_REF" fi -if [ "$REF" =~ "main" ]; then +if [[ "$REF" =~ "main" ]]; then DIR=dev else # Strip off .X From 1cb3944f8f510cc2d50ce139031749429d5a2061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 21:40:27 +0200 Subject: [PATCH 20/90] [DBG] Debugging paths. --- build_tools/circle/push_doc.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index a03426769c3d8..02f5a2c00fde4 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -9,13 +9,15 @@ cd scikit-learn.github.io DOC_REPO="scikit-learn.github.io" -if [[ -z "$GITHUB_BASE_REF" ]]; then +if [[ -z "$GITHUB_BASE_REF" ]] +then REF="$GITHUB_REF" else REF="$GITHUB_BASE_REF" fi -if [[ "$REF" =~ "main" ]]; then +if [[ "$REF" =~ "main" ]] +then DIR=dev else # Strip off .X @@ -26,7 +28,6 @@ MSG="Pushing the docs to $DIR/ for branch: $REF, commit $GITHUB_SHA" # check if it's a new branch echo $DIR > .git/info/sparse-checkout - if ! git show HEAD:$DIR >/dev/null then # directory does not exist. Need to make it so sparse checkout works @@ -41,6 +42,8 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi +ls -l $GITHUB_WORKSPACE + cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" From 919788c806314d14c54b77ebe750868095a8d61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 22:01:26 +0200 Subject: [PATCH 21/90] [DBG] Debugging paths. --- .github/workflows/docs.yml | 6 +----- build_tools/circle/push_doc.sh | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3d22874946b1b..837e7b356800d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,11 +5,6 @@ on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" - push: - branches: - - main - # Release branches - - "[0-9]+.[0-9]+.X" pull_request: branches: - main @@ -60,6 +55,7 @@ jobs: uses: actions/download-artifact@v2 with: name: docs + path: docs - name: Checkout scikit-learn uses: actions/checkout@v2 diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 02f5a2c00fde4..c33e07d08235b 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -42,8 +42,6 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi -ls -l $GITHUB_WORKSPACE - cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" From 16e52044b70d87408db94eae219326633bfc4750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 22:20:26 +0200 Subject: [PATCH 22/90] [DBG] Debugging. --- .github/workflows/docs.yml | 2 ++ build_tools/circle/push_doc.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 837e7b356800d..bc742df6500e6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -50,6 +50,7 @@ jobs: push: runs-on: ubuntu-latest needs: [build] + # if: github.event_name != 'pull_request' steps: - name: Download documentation uses: actions/download-artifact@v2 @@ -67,6 +68,7 @@ jobs: with: repository: scikit-learn/scikit-learn.github.io path: scikit-learn.github.io + # ssh-key: ${{ secrets.SKLEARN_CI_SSH_KEY }} - name: Push documentation run: scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index c33e07d08235b..5561d86c669e3 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -42,6 +42,8 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi +ls -l $GITHUB_WORKSPACE/docs + cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" From 739e22de8d08ed33b7e6535382dc54ebdba87628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 22:21:11 +0200 Subject: [PATCH 23/90] [DBG] Debugging. --- build_tools/circle/push_doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 5561d86c669e3..0496feb69c543 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -52,6 +52,6 @@ git config push.default matching git add -f $DIR/ git commit -m "$MSG" $DIR -git push +# git push echo $MSG From c2112834c42ec8877c5c2c4df6eaae9fa0c733b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 24 Sep 2021 22:40:46 +0200 Subject: [PATCH 24/90] [DBG] Debugging. --- .github/workflows/docs.yml | 2 +- build_tools/circle/push_doc.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bc742df6500e6..0dd65224db075 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn - uses: actions/checkout@v2 + uses: actions/checkout@v1 - name: Setup Python uses: actions/setup-python@v2 diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 0496feb69c543..c1979df28d77d 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -42,8 +42,6 @@ then git rm -rf $DIR/ && rm -rf $DIR/ fi -ls -l $GITHUB_WORKSPACE/docs - cp -R $GITHUB_WORKSPACE/docs $DIR git config user.email "olivier.grisel+sklearn-ci@gmail.com" From 6c1d5b4594369918cab35ab2b1434d0427093dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:15:28 +0200 Subject: [PATCH 25/90] [CI] Monkey-patch build_doc.sh to work with both documentation builders --- .circleci/config.yml | 15 +++- .github/workflows/docs.yml | 38 ++-------- build_tools/circle/push_doc.sh | 68 ++++++++++------- build_tools/{circle => github}/build_doc.sh | 84 ++++++++++++++++++--- 4 files changed, 132 insertions(+), 73 deletions(-) rename build_tools/{circle => github}/build_doc.sh (69%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91599e8d05bda..1410fc2ad49ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: keys: - doc-min-deps-ccache-{{ .Branch }} - doc-min-deps-ccache - - run: ./build_tools/circle/build_doc.sh + - run: ./build_tools/github/build_doc.sh - save_cache: key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} paths: @@ -76,7 +76,7 @@ jobs: keys: - doc-ccache-{{ .Branch }} - doc-ccache - - run: ./build_tools/circle/build_doc.sh + - run: ./build_tools/github/build_doc.sh - save_cache: key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} paths: @@ -159,9 +159,18 @@ jobs: workflows: version: 2 - lint: + build-doc-and-deploy: jobs: - lint + - doc: + requires: + - lint + - doc-min-dependencies: + requires: + - lint + - deploy: + requires: + - doc linux-arm64: jobs: - linux-arm64 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0dd65224db075..5ea3b4cd2bc46 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,12 +5,10 @@ on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" + # TODO: Remove when the pull request is ready to merge pull_request: branches: - main - - "[0-9]+.[0-9]+.X" - # Manual run - workflow_dispatch: jobs: # Build the documentation @@ -24,16 +22,19 @@ jobs: uses: actions/setup-python@v2 - name: Build documentation - run: build_tools/circle/build_doc.sh + run: build_tools/github/build_doc.sh env: OMP_NUM_THREADS: 2 MKL_NUM_THREADS: 2 + CONDA_ENV_NAME: testenv NUMPY_VERSION: latest SCIPY_VERSION: latest MATPLOTLIB_VERSION: latest CYTHON_VERSION: latest SCIKIT_IMAGE_VERSION: latest - SPHINX_VERSION: min + # Bump the version from time to time because the + # latest version tends to break things too often + SPHINX_VERSION: 4.2.0 PANDAS_VERSION: latest SPHINX_GALLERY_VERSION: latest NUMPYDOC_VERSION: latest @@ -45,30 +46,3 @@ jobs: with: name: docs path: doc/_build/html/stable - - # Push the documentation - push: - runs-on: ubuntu-latest - needs: [build] - # if: github.event_name != 'pull_request' - steps: - - name: Download documentation - uses: actions/download-artifact@v2 - with: - name: docs - path: docs - - - name: Checkout scikit-learn - uses: actions/checkout@v2 - with: - path: scikit-learn - - - name: Checkout scikit-learn.github.io - uses: actions/checkout@v2 - with: - repository: scikit-learn/scikit-learn.github.io - path: scikit-learn.github.io - # ssh-key: ${{ secrets.SKLEARN_CI_SSH_KEY }} - - - name: Push documentation - run: scikit-learn/build_tools/circle/push_doc.sh diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index c1979df28d77d..5b94211e4e30e 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -1,55 +1,65 @@ #!/bin/bash - -# The behavior of the script is controlled by environment variable -# defined in the docs.yml in the actions level folder of the project. +# This script is meant to be called in the "deploy" step defined in +# circle.yml. See https://circleci.com/docs/ for more details. +# The behavior of the script is controlled by environment variable defined +# in the circle.yml in the top level folder of the project. set -ex -cd scikit-learn.github.io +if [ -z $CIRCLE_PROJECT_USERNAME ]; +then USERNAME="sklearn-ci"; +else USERNAME=$CIRCLE_PROJECT_USERNAME; +fi DOC_REPO="scikit-learn.github.io" +GENERATED_DOC_DIR=$1 -if [[ -z "$GITHUB_BASE_REF" ]] -then - REF="$GITHUB_REF" -else - REF="$GITHUB_BASE_REF" +if [[ -z "$GENERATED_DOC_DIR" ]]; then + echo "Need to pass directory of the generated doc as argument" + echo "Usage: $0 " + exit 1 fi -if [[ "$REF" =~ "main" ]] +# Absolute path needed because we use cd further down in this script +GENERATED_DOC_DIR=$(readlink -f $GENERATED_DOC_DIR) + +if [ "$CIRCLE_BRANCH" = "main" ] then - DIR=dev + dir=dev else # Strip off .X - DIR="${REF::-2}" + dir="${CIRCLE_BRANCH::-2}" fi -MSG="Pushing the docs to $DIR/ for branch: $REF, commit $GITHUB_SHA" +MSG="Pushing the docs to $dir/ for branch: $CIRCLE_BRANCH, commit $CIRCLE_SHA1" + +cd $HOME +if [ ! -d $DOC_REPO ]; +then git clone --depth 1 --no-checkout "git@github.com:scikit-learn/"$DOC_REPO".git"; +fi +cd $DOC_REPO # check if it's a new branch -echo $DIR > .git/info/sparse-checkout -if ! git show HEAD:$DIR >/dev/null + +echo $dir > .git/info/sparse-checkout +if ! git show HEAD:$dir >/dev/null then # directory does not exist. Need to make it so sparse checkout works - mkdir $DIR - touch $DIR/index.html - git add $DIR + mkdir $dir + touch $dir/index.html + git add $dir fi git checkout main git reset --hard origin/main -if [ -d $DIR ] +if [ -d $dir ] then - git rm -rf $DIR/ && rm -rf $DIR/ + git rm -rf $dir/ && rm -rf $dir/ fi - -cp -R $GITHUB_WORKSPACE/docs $DIR - +cp -R $GENERATED_DOC_DIR $dir git config user.email "olivier.grisel+sklearn-ci@gmail.com" -git config user.name "sklearn-ci" +git config user.name $USERNAME git config push.default matching - -git add -f $DIR/ -git commit -m "$MSG" $DIR -# git push - +git add -f $dir/ +git commit -m "$MSG" $dir +git push echo $MSG diff --git a/build_tools/circle/build_doc.sh b/build_tools/github/build_doc.sh similarity index 69% rename from build_tools/circle/build_doc.sh rename to build_tools/github/build_doc.sh index fd39a3597d81c..d3194da88b08d 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -17,16 +17,25 @@ set -e # If the inspection of the current commit fails for any reason, the default # behavior is to quick build the documentation. +if [! -z "$GITHUB_ACTION" ] +then + # Map the variables for the new documentation builder to the old one + CIRCLE_SHA1 = $GITHUB_SHA + CIRCLE_BRANCH = $GITHUB_REF + CIRCLE_JOB = $GITHUB_JOB + CI_PULL_REQUEST = [ "$GITHUB_EVENT_NAME" == pull_request ] && echo true +fi + get_build_type() { - if [ -z "$GITHUB_SHA" ] + if [ -z "$CIRCLE_SHA1" ] then - echo SKIP: undefined GITHUB_SHA + echo SKIP: undefined CIRCLE_SHA1 return fi - commit_msg=$(git log --format=%B -n 1 $GITHUB_SHA) + commit_msg=$(git log --format=%B -n 1 $CIRCLE_SHA1) if [ -z "$commit_msg" ] then - echo QUICK BUILD: failed to inspect commit $GITHUB_SHA + echo QUICK BUILD: failed to inspect commit $CIRCLE_SHA1 return fi if [[ "$commit_msg" =~ \[doc\ skip\] ]] @@ -44,12 +53,12 @@ get_build_type() { echo BUILD: [doc build] marker found return fi - if [[ "$GITHUB_EVENT_NAME" != pull_request ]] + if [ -z "$CI_PULL_REQUEST" ] then echo BUILD: not a pull request return fi - git_range="origin/main...$GITHUB_SHA" + git_range="origin/main...$CIRCLE_SHA1" git fetch origin main >&2 || (echo QUICK BUILD: failed to get changed filenames for $git_range; return) filenames=$(git diff --name-only $git_range) if [ -z "$filenames" ] @@ -114,7 +123,7 @@ then exit 0 fi -if [[ "$GITHUB_REF" =~ ^main$|^[0-9]+\.[0-9]+\.X$ && "$GITHUB_EVENT_NAME" != pull_request ]] +if [[ "$CIRCLE_BRANCH" =~ ^main$|^[0-9]+\.[0-9]+\.X$ && -z "$CI_PULL_REQUEST" ]] then # ZIP linked into HTML make_args=dist @@ -190,15 +199,72 @@ python setup.py develop export OMP_NUM_THREADS=1 -if [[ "$GITHUB_SHA" =~ ^main$ && "$GITHUB_EVENT_NAME" == pull_request ]] +if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]] then # List available documentation versions if on main python build_tools/circle/list_versions.py > doc/versions.rst fi # The pipefail is requested to propagate exit code -cd doc && make $make_args +set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt # Insert the version warning for deployment find _build/html/stable -name "*.html" | xargs sed -i '/<\/body>/ i \ \ ' + +cd - +set +o pipefail + +affected_doc_paths() { + files=$(git diff --name-only origin/main...$CIRCLE_SHA1) + echo "$files" | grep ^doc/.*\.rst | sed 's/^doc\/\(.*\)\.rst$/\1.html/' + echo "$files" | grep ^examples/.*.py | sed 's/^\(.*\)\.py$/auto_\1.html/' + sklearn_files=$(echo "$files" | grep '^sklearn/') + if [ -n "$sklearn_files" ] + then + grep -hlR -f<(echo "$sklearn_files" | sed 's/^/scikit-learn\/blob\/[a-z0-9]*\//') doc/_build/html/stable/modules/generated | cut -d/ -f5- + fi +} + +affected_doc_warnings() { + files=$(git diff --name-only origin/main...$CIRCLE_SHA1) + # Look for sphinx warnings only in files affected by the PR + if [ -n "$files" ] + then + for af in ${files[@]} + do + warn+=`grep WARNING ~/log.txt | grep $af` + done + fi + echo "$warn" +} + +if [ -n "$CI_PULL_REQUEST" ] +then + echo "The following documentation warnings may have been generated by PR #$CI_PULL_REQUEST:" + warnings=$(affected_doc_warnings) + if [ -z "$warnings" ] + then + warnings="/home/circleci/project/ no warnings" + fi + echo "$warnings" + + echo "The following documentation files may have been changed by PR #$CI_PULL_REQUEST:" + affected=$(affected_doc_paths) + echo "$affected" + ( + echo '
    ' + echo "$affected" | sed 's|.*|
  • & [dev, stable]
    • |' + echo '

General: Home | API Reference | Examples

' + echo 'Sphinx Warnings in affected files
    ' + echo "$warnings" | sed 's/\/home\/circleci\/project\//
  • /g' + echo '
' + ) > 'doc/_build/html/stable/_changed.html' + + if [ "$warnings" != "/home/circleci/project/ no warnings" ] + then + echo "Sphinx generated warnings when building the documentation related to files modified in this PR." + echo "Please check doc/_build/html/stable/_changed.html" + exit 1 + fi +fi From 905b9e085e5acde22f931a3afc90df796cc2225f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:18:21 +0200 Subject: [PATCH 26/90] [FIX] Fix syntax --- build_tools/github/build_doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index d3194da88b08d..08e27e9a5d6cf 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -17,13 +17,13 @@ set -e # If the inspection of the current commit fails for any reason, the default # behavior is to quick build the documentation. -if [! -z "$GITHUB_ACTION" ] +if [[! -z "$GITHUB_ACTION" ]] then # Map the variables for the new documentation builder to the old one CIRCLE_SHA1 = $GITHUB_SHA CIRCLE_BRANCH = $GITHUB_REF CIRCLE_JOB = $GITHUB_JOB - CI_PULL_REQUEST = [ "$GITHUB_EVENT_NAME" == pull_request ] && echo true + CI_PULL_REQUEST = [[ "$GITHUB_EVENT_NAME" == pull_request ]] && echo true fi get_build_type() { From 015af425fcec4dc784525d499e88dbe104b6f5d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:19:54 +0200 Subject: [PATCH 27/90] [FIX] Fix syntax --- build_tools/github/build_doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index 08e27e9a5d6cf..dcd78131ff02e 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -17,13 +17,13 @@ set -e # If the inspection of the current commit fails for any reason, the default # behavior is to quick build the documentation. -if [[! -z "$GITHUB_ACTION" ]] +if [ -n "$GITHUB_ACTION" ] then # Map the variables for the new documentation builder to the old one CIRCLE_SHA1 = $GITHUB_SHA CIRCLE_BRANCH = $GITHUB_REF CIRCLE_JOB = $GITHUB_JOB - CI_PULL_REQUEST = [[ "$GITHUB_EVENT_NAME" == pull_request ]] && echo true + CI_PULL_REQUEST = [ "$GITHUB_EVENT_NAME" == pull_request ] && echo true fi get_build_type() { From a006f95d4c302e20a8bf5722597a10add05f11bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:22:42 +0200 Subject: [PATCH 28/90] [FIX] Fix syntax --- build_tools/github/build_doc.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index dcd78131ff02e..487e54a986cd0 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -20,9 +20,9 @@ set -e if [ -n "$GITHUB_ACTION" ] then # Map the variables for the new documentation builder to the old one - CIRCLE_SHA1 = $GITHUB_SHA - CIRCLE_BRANCH = $GITHUB_REF - CIRCLE_JOB = $GITHUB_JOB + CIRCLE_SHA1 = "$GITHUB_SHA" + CIRCLE_BRANCH = "$GITHUB_REF" + CIRCLE_JOB = "$GITHUB_JOB" CI_PULL_REQUEST = [ "$GITHUB_EVENT_NAME" == pull_request ] && echo true fi From 4bd04ab077c0bad518625347c4171985a16715a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:24:56 +0200 Subject: [PATCH 29/90] [FIX] Fix syntax --- build_tools/github/build_doc.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index 487e54a986cd0..b3d08f8f62af5 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -20,10 +20,15 @@ set -e if [ -n "$GITHUB_ACTION" ] then # Map the variables for the new documentation builder to the old one - CIRCLE_SHA1 = "$GITHUB_SHA" - CIRCLE_BRANCH = "$GITHUB_REF" - CIRCLE_JOB = "$GITHUB_JOB" - CI_PULL_REQUEST = [ "$GITHUB_EVENT_NAME" == pull_request ] && echo true + CIRCLE_SHA1=$GITHUB_SHA + CIRCLE_BRANCH=$GITHUB_REF + CIRCLE_JOB=$GITHUB_JOB + CI_PULL_REQUEST=[ "$GITHUB_EVENT_NAME" == pull_request ] && echo true + + echo $CIRCLE_SHA1 + echo $CIRCLE_BRANCH + echo $CIRCLE_JOB + echo $CI_PULL_REQUEST fi get_build_type() { From 65e01be414b723df7039c0fdb02f9da9df5d7716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 15:53:56 +0200 Subject: [PATCH 30/90] [FIX] Checkout correct commit --- .github/workflows/docs.yml | 4 +++- build_tools/github/build_doc.sh | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5ea3b4cd2bc46..49c21cb4d9d03 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Setup Python uses: actions/setup-python@v2 diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index b3d08f8f62af5..7e542da509959 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -24,11 +24,6 @@ then CIRCLE_BRANCH=$GITHUB_REF CIRCLE_JOB=$GITHUB_JOB CI_PULL_REQUEST=[ "$GITHUB_EVENT_NAME" == pull_request ] && echo true - - echo $CIRCLE_SHA1 - echo $CIRCLE_BRANCH - echo $CIRCLE_JOB - echo $CI_PULL_REQUEST fi get_build_type() { From b240998ca585fac87a50b374906b035ae8fdf14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 16:22:04 +0200 Subject: [PATCH 31/90] [FIX] Checkout correct commit --- build_tools/github/build_doc.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index 7e542da509959..9a7dda63e9db6 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -20,10 +20,14 @@ set -e if [ -n "$GITHUB_ACTION" ] then # Map the variables for the new documentation builder to the old one - CIRCLE_SHA1=$GITHUB_SHA - CIRCLE_BRANCH=$GITHUB_REF + CIRCLE_SHA1=$(git log --no-merges -1 --pretty=format:%H) + CIRCLE_BRANCH=$GITHUB_REF_NAME CIRCLE_JOB=$GITHUB_JOB - CI_PULL_REQUEST=[ "$GITHUB_EVENT_NAME" == pull_request ] && echo true + + if [ "$GITHUB_EVENT_NAME" == "pull_request" ] + then + CI_PULL_REQUEST=true + fi fi get_build_type() { @@ -32,7 +36,7 @@ get_build_type() { echo SKIP: undefined CIRCLE_SHA1 return fi - commit_msg=$(git log --format=%B -n 1 $CIRCLE_SHA1) + commit_msg=$(git log --no-merges -1 --oneline $CIRCLE_SHA1) if [ -z "$commit_msg" ] then echo QUICK BUILD: failed to inspect commit $CIRCLE_SHA1 From 20e56b2e450aacc7e1392464c0a58523a7625d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 16:42:04 +0200 Subject: [PATCH 32/90] [FIX] Checkout correct commit --- build_tools/github/build_doc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index 9a7dda63e9db6..197392d69b0b2 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -21,12 +21,14 @@ if [ -n "$GITHUB_ACTION" ] then # Map the variables for the new documentation builder to the old one CIRCLE_SHA1=$(git log --no-merges -1 --pretty=format:%H) - CIRCLE_BRANCH=$GITHUB_REF_NAME CIRCLE_JOB=$GITHUB_JOB if [ "$GITHUB_EVENT_NAME" == "pull_request" ] then + CIRCLE_BRANCH=$GITHUB_HEAD_REF CI_PULL_REQUEST=true + else + CIRCLE_BRANCH=$GITHUB_REF_NAME fi fi From 919e611b960cb0b55c9f7685074c683d5d5ef22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Wed, 27 Apr 2022 16:53:34 +0200 Subject: [PATCH 33/90] [CLN] Remove pull request trigger --- .github/workflows/docs.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 49c21cb4d9d03..4546de29d33a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,10 +5,6 @@ on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" - # TODO: Remove when the pull request is ready to merge - pull_request: - branches: - - main jobs: # Build the documentation From 46156853349c13f0cbaa6a2a866d323817799fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 10:08:47 +0200 Subject: [PATCH 34/90] [FEA] Add doc-min-dependencies job --- .github/workflows/docs.yml | 47 ++++++++++++++++++++++++++++++--- build_tools/github/build_doc.sh | 2 +- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4546de29d33a0..297e4af6ab4a1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,10 +5,51 @@ on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" + # TODO: Remove when the pull request is ready to merge + pull_request: + branches: + - main jobs: - # Build the documentation - build: + # Build the documentation against the minimum version of the dependencies + doc-min-dependencies: + runs-on: ubuntu-latest + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build documentation + run: build_tools/github/build_doc.sh + env: + OMP_NUM_THREADS: 2 + MKL_NUM_THREADS: 2 + CONDA_ENV_NAME: testenv + PYTHON_VERSION: 3.8 + NUMPY_VERSION: min + SCIPY_VERSION: min + MATPLOTLIB_VERSION: min + CYTHON_VERSION: min + SCIKIT_IMAGE_VERSION: min + SPHINX_VERSION: min + PANDAS_VERSION: min + SPHINX_GALLERY_VERSION: min + NUMPYDOC_VERSION: min + SPHINX_PROMPT_VERSION: min + SPHINXEXT_OPENGRAPH_VERSION: min + + - name: Upload documentation + uses: actions/upload-artifact@v2 + with: + name: doc-min-dependencies + path: doc/_build/html/stable + + # Build the documentation against the latest version of the dependencies + doc: runs-on: ubuntu-latest steps: - name: Checkout scikit-learn @@ -42,5 +83,5 @@ jobs: - name: Upload documentation uses: actions/upload-artifact@v2 with: - name: docs + name: doc path: doc/_build/html/stable diff --git a/build_tools/github/build_doc.sh b/build_tools/github/build_doc.sh index 197392d69b0b2..7fbccadd738ed 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/github/build_doc.sh @@ -38,7 +38,7 @@ get_build_type() { echo SKIP: undefined CIRCLE_SHA1 return fi - commit_msg=$(git log --no-merges -1 --oneline $CIRCLE_SHA1) + commit_msg=$(git log --format=%B -n 1 $CIRCLE_SHA1) if [ -z "$commit_msg" ] then echo QUICK BUILD: failed to inspect commit $CIRCLE_SHA1 From 5463071ef5d12af0a39ceb625358e1949b58f81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 10:45:51 +0200 Subject: [PATCH 35/90] [CLN] Remove pull request trigger --- .github/workflows/docs.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 297e4af6ab4a1..3e4f8d550b375 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,10 +5,6 @@ on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" - # TODO: Remove when the pull request is ready to merge - pull_request: - branches: - - main jobs: # Build the documentation against the minimum version of the dependencies From 16d57edb90764fc9c567f3317561803641c4d63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:37:15 +0200 Subject: [PATCH 36/90] [FEA] First attempt to trigger using actions --- .circleci/config.yml | 125 +++++------------- .../workflows/{docs.yml => build-docs.yml} | 11 +- .github/workflows/push-docs.yml | 19 +++ build_tools/circle/download_documentation.sh | 7 + 4 files changed, 67 insertions(+), 95 deletions(-) rename .github/workflows/{docs.yml => build-docs.yml} (92%) create mode 100644 .github/workflows/push-docs.yml create mode 100644 build_tools/circle/download_documentation.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 1410fc2ad49ac..cccb62620877c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,116 +1,54 @@ version: 2.1 +# Parameters variables required to conditionally execute the "deploy" job and +parameters: + # The event triggering the pipeline + GHA_Event: + type: string + default: "" + # The user triggering the pipeline + GHA_Actor: + type: string + default: "" + # The identifier assigned in the workflow configuration + GHA_Action: + type: string + default: "" + # The metadata given as input parameters + GHA_Meta: + type: string + default: "" + jobs: doc-min-dependencies: docker: - image: cimg/python:3.8.12 environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.8 - - NUMPY_VERSION: 'min' - - SCIPY_VERSION: 'min' - - MATPLOTLIB_VERSION: 'min' - - CYTHON_VERSION: 'min' - - SCIKIT_IMAGE_VERSION: 'min' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'min' - - SPHINX_GALLERY_VERSION: 'min' - - NUMPYDOC_VERSION: 'min' - - SPHINX_PROMPT_VERSION: 'min' - - SPHINXEXT_OPENGRAPH_VERSION: 'min' + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_META >>/doc-min-dependencies.zip steps: - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-doc-min-deps-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-min-deps-ccache-{{ .Branch }} - - doc-min-deps-ccache - - run: ./build_tools/github/build_doc.sh - - save_cache: - key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-doc-min-deps-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data + - run: ./build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt doc: docker: - image: cimg/python:3.8.12 environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: '3.9' - - NUMPY_VERSION: 'latest' - - SCIPY_VERSION: 'latest' - - MATPLOTLIB_VERSION: 'latest' - - CYTHON_VERSION: 'latest' - - SCIKIT_IMAGE_VERSION: 'latest' - # Bump the sphinx version from time to time. Avoid latest sphinx version - # that tends to break things slightly too often - - SPHINX_VERSION: 4.2.0 - - PANDAS_VERSION: 'latest' - - SPHINX_GALLERY_VERSION: 'latest' - - NUMPYDOC_VERSION: 'latest' - - SPHINX_PROMPT_VERSION: 'latest' - - SPHINXEXT_OPENGRAPH_VERSION: 'latest' + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_META >>/doc.zip steps: - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-doc-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-ccache-{{ .Branch }} - - doc-ccache - - run: ./build_tools/github/build_doc.sh - - save_cache: - key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-doc-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data + - run: ./build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt - # Persists generated documentation so that it can be attached and deployed - # in the 'deploy' step. + # Persists the generated documentation, so that it + # can be attached and deployed in the "deploy" job - persist_to_workspace: root: doc/_build/html paths: . - lint: - docker: - - image: cimg/python:3.8.12 - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - run: - name: dependencies - command: pip install flake8 - - run: - name: linting - command: ./build_tools/circle/linting.sh - linux-arm64: machine: image: ubuntu-2004:202101-01 @@ -160,17 +98,16 @@ jobs: workflows: version: 2 build-doc-and-deploy: + # This workflow is executed conditionally upon the + # action is triggered to ignore normal push events + when: << pipeline.parameters.GHA_Action >> jobs: - - lint - - doc: - requires: - - lint - - doc-min-dependencies: - requires: - - lint + - doc-min-dependencies + - doc - deploy: requires: - doc + linux-arm64: jobs: - linux-arm64 diff --git a/.github/workflows/docs.yml b/.github/workflows/build-docs.yml similarity index 92% rename from .github/workflows/docs.yml rename to .github/workflows/build-docs.yml index 3e4f8d550b375..8e82661f6b4fd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/build-docs.yml @@ -1,10 +1,19 @@ -# Workflow to build and publish the documentation +# Workflow to build the documentation name: Documentation builder on: schedule: # Nightly build at 3:42 A.M. - cron: "42 3 */1 * *" + push: + branches: + - main + # Release branches + - "[0-9]+.[0-9]+.X" + pull_request: + branches: + - main + - "[0-9]+.[0-9]+.X" jobs: # Build the documentation against the minimum version of the dependencies diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml new file mode 100644 index 0000000000000..3d09cdc92d46a --- /dev/null +++ b/.github/workflows/push-docs.yml @@ -0,0 +1,19 @@ +# Workflow to trigger the jobs that will host the documentation +name: Documentation push trigger +on: + # Run the workflow after the separate "Documentation builder" workflow completes + workflows: [Documentation builder] + types: + - completed + +jobs: + push: + runs-on: ubuntu-latest + # Run the job only if the "Documentation builder" workflow succeeded + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Trigger hosting jobs + uses: CircleCI-Public/trigger-circleci-pipeline-action@v1.0.4 + env: + GHA_META: https://nightly.link/scikit-learn/scikit-learn/actions/runs/${{ github.run_id }} + CCI_TOKEN: ${{ secrets.CCI_TOKEN }} diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh new file mode 100644 index 0000000000000..8c46a8c55bc08 --- /dev/null +++ b/build_tools/circle/download_documentation.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e +set -x + +wget $GHA_META +unzip doc.zip -d doc From 466d00b4325db33db1abbdc8b04d06b0b85db8a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:40:43 +0200 Subject: [PATCH 37/90] [FIX] Fix syntax and variables --- .circleci/config.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cccb62620877c..41ddf0e4d05c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_META >>/doc-min-dependencies.zip + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_Meta >>/doc-min-dependencies.zip steps: - checkout - run: ./build_tools/circle/download_documentation.sh @@ -36,7 +36,7 @@ jobs: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_META >>/doc.zip + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_Meta >>/doc.zip steps: - checkout - run: ./build_tools/circle/download_documentation.sh @@ -97,17 +97,18 @@ jobs: workflows: version: 2 + + linux-arm64: + jobs: + - linux-arm64 + + when: << pipeline.parameters.GHA_Action >> build-doc-and-deploy: # This workflow is executed conditionally upon the # action is triggered to ignore normal push events - when: << pipeline.parameters.GHA_Action >> jobs: - doc-min-dependencies - doc - deploy: requires: - doc - - linux-arm64: - jobs: - - linux-arm64 From 990c30495f19edd6ec11a19f2cc1dd0253079952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:42:04 +0200 Subject: [PATCH 38/90] [FIX] Fix syntax and variables --- .github/workflows/push-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 3d09cdc92d46a..817cde696e478 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -1,8 +1,8 @@ # Workflow to trigger the jobs that will host the documentation name: Documentation push trigger on: - # Run the workflow after the separate "Documentation builder" workflow completes workflows: [Documentation builder] + # Run the workflow after the separate "Documentation builder" workflow completes types: - completed From a854c3df29d76f00ac222d262413d7932b13a78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:43:04 +0200 Subject: [PATCH 39/90] [FIX] Fix syntax and variables --- .github/workflows/push-docs.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 817cde696e478..b1d2ab61ebe2a 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -1,10 +1,11 @@ # Workflow to trigger the jobs that will host the documentation name: Documentation push trigger on: - workflows: [Documentation builder] + workflow_run: # Run the workflow after the separate "Documentation builder" workflow completes - types: - - completed + workflows: [Documentation builder] + types: + - completed jobs: push: From d3c4bf60076f987f3559bd0715f8d4806f2de212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:45:25 +0200 Subject: [PATCH 40/90] [FIX] Fix syntax and variables --- .github/workflows/push-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index b1d2ab61ebe2a..182259ed78455 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -4,8 +4,8 @@ on: workflow_run: # Run the workflow after the separate "Documentation builder" workflow completes workflows: [Documentation builder] - types: - - completed + types: + - completed jobs: push: From 97d5370bb97a44189967b3d43609565e9041b4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 18:51:15 +0200 Subject: [PATCH 41/90] [FIX] Fix syntax and variables --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41ddf0e4d05c9..229ff78554d62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,12 +98,8 @@ jobs: workflows: version: 2 - linux-arm64: - jobs: - - linux-arm64 - - when: << pipeline.parameters.GHA_Action >> build-doc-and-deploy: + when: << pipeline.parameters.GHA_Action >> # This workflow is executed conditionally upon the # action is triggered to ignore normal push events jobs: @@ -112,3 +108,7 @@ workflows: - deploy: requires: - doc + + linux-arm64: + jobs: + - linux-arm64 From 15d9746835cd9ae6637769ee10fb9d8958c94fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 19:17:29 +0200 Subject: [PATCH 42/90] Just for testing --- .github/workflows/build-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8e82661f6b4fd..61f89a0698eee 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -8,6 +8,7 @@ on: push: branches: - main + - move_doc_builder # Release branches - "[0-9]+.[0-9]+.X" pull_request: From dd7308edfc879e59a70813eab272153926c5c688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 20:31:15 +0200 Subject: [PATCH 43/90] Trigger build From 07533badfddd744dc3fa99cb9d519b89a14b69b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 21:14:08 +0200 Subject: [PATCH 44/90] Trigger build --- .github/workflows/build-docs.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 61f89a0698eee..71ba7f7b6821c 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -2,13 +2,9 @@ name: Documentation builder on: - schedule: - # Nightly build at 3:42 A.M. - - cron: "42 3 */1 * *" push: branches: - main - - move_doc_builder # Release branches - "[0-9]+.[0-9]+.X" pull_request: From c08e4ef124d7092878c0c9ba51898802e915aa42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 21:34:22 +0200 Subject: [PATCH 45/90] More testing --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 229ff78554d62..0109c0e611813 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,6 +109,6 @@ workflows: requires: - doc - linux-arm64: - jobs: - - linux-arm64 + # linux-arm64: + # jobs: + # - linux-arm64 From 65f0644967cefb5a84a4b57ebe2bf2a7a1ab9f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:28:06 +0200 Subject: [PATCH 46/90] More testing --- .circleci/config.yml | 37 ++++++++++----------------- .github/workflows/push-docs.yml | 7 +++-- build_tools/github/trigger_hosting.sh | 12 +++++++++ 3 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 build_tools/github/trigger_hosting.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 0109c0e611813..5adaa43143476 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,30 +1,17 @@ version: 2.1 -# Parameters variables required to conditionally execute the "deploy" job and +# Parameters required to conditionally execute the "deploy" job and parameters: - # The event triggering the pipeline - GHA_Event: + GITHUB_RUN_URL: type: string - default: "" - # The user triggering the pipeline - GHA_Actor: - type: string - default: "" - # The identifier assigned in the workflow configuration - GHA_Action: - type: string - default: "" - # The metadata given as input parameters - GHA_Meta: - type: string - default: "" + default: "none" jobs: doc-min-dependencies: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_Meta >>/doc-min-dependencies.zip + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_RUL >>/doc-min-dependencies.zip steps: - checkout - run: ./build_tools/circle/download_documentation.sh @@ -36,7 +23,7 @@ jobs: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GHA_Meta >>/doc.zip + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip steps: - checkout - run: ./build_tools/circle/download_documentation.sh @@ -99,9 +86,11 @@ workflows: version: 2 build-doc-and-deploy: - when: << pipeline.parameters.GHA_Action >> - # This workflow is executed conditionally upon the - # action is triggered to ignore normal push events + when: + not: + equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] + # The following jobs will be executed conditionally upon + # the action is triggered to ignore normal push events jobs: - doc-min-dependencies - doc @@ -109,6 +98,6 @@ workflows: requires: - doc - # linux-arm64: - # jobs: - # - linux-arm64 + linux-arm64: + jobs: + - linux-arm64 diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 182259ed78455..99cf4f03524d1 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -13,8 +13,11 @@ jobs: # Run the job only if the "Documentation builder" workflow succeeded if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: + - name: Checkout scikit-learn + uses: actions/checkout@v2 + - name: Trigger hosting jobs - uses: CircleCI-Public/trigger-circleci-pipeline-action@v1.0.4 + run: build_tools/github/trigger_hosting.sh env: - GHA_META: https://nightly.link/scikit-learn/scikit-learn/actions/runs/${{ github.run_id }} + GITHUB_RUN_URL: https://nightly.link/scikit-learn/scikit-learn/actions/runs/${{ github.run_id }} CCI_TOKEN: ${{ secrets.CCI_TOKEN }} diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh new file mode 100644 index 0000000000000..81fa6b3880449 --- /dev/null +++ b/build_tools/github/trigger_hosting.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e +set -x + +curl --request POST \ + --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/continue \ + --header "Circle-Token: $CCI_TOKEN" \ + --header "content-type: application/json" \ + --header "x-attribution-actor-id: github_actions" \ + --header "x-attribution-login: github_actions" \ + --data \{\"parameters\":\{\"GITHUB_RUN_URL\":\"$GITHUB_RUN_URL\"\}\} From fe0a46e6cf3bc3a192f593f06ca1963a0673a81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:29:19 +0200 Subject: [PATCH 47/90] More testing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5adaa43143476..3d0e6d2061a28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,8 +89,8 @@ workflows: when: not: equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] - # The following jobs will be executed conditionally upon - # the action is triggered to ignore normal push events + # The following jobs will be executed only if the + # action is executed to ignore normal push events jobs: - doc-min-dependencies - doc From 2e59ae9ac9fd593074b85aba040e69121e0620dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:30:18 +0200 Subject: [PATCH 48/90] [FIX] Fix parameter name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d0e6d2061a28..1677648a4432c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_RUL >>/doc-min-dependencies.zip + - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip steps: - checkout - run: ./build_tools/circle/download_documentation.sh From 5825d19be8f0810d946f540c4a555c57c6c2730e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:35:40 +0200 Subject: [PATCH 49/90] Trigger build From 1ee28de2c6c92b0da23509ac86f6c6f4db3a7060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:50:36 +0200 Subject: [PATCH 50/90] Trigger build From d45506b1f75e16db8bc2ad500867da7469366660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:56:15 +0200 Subject: [PATCH 51/90] [FIX] Fix run in push-docs --- .github/workflows/push-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 99cf4f03524d1..2982e6ce5536a 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v2 - name: Trigger hosting jobs - run: build_tools/github/trigger_hosting.sh + run: bash build_tools/github/trigger_hosting.sh env: GITHUB_RUN_URL: https://nightly.link/scikit-learn/scikit-learn/actions/runs/${{ github.run_id }} CCI_TOKEN: ${{ secrets.CCI_TOKEN }} From 211f69f96b15f8e0108c2c310dfc7f743f51daaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 22:57:44 +0200 Subject: [PATCH 52/90] Trigger build From 7dd50c8e1accd93fc808de3322d500695706fe16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 23:12:42 +0200 Subject: [PATCH 53/90] [FIX] Fix run in push-docs --- .github/workflows/push-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 2982e6ce5536a..11c56209cfd23 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -19,5 +19,5 @@ jobs: - name: Trigger hosting jobs run: bash build_tools/github/trigger_hosting.sh env: - GITHUB_RUN_URL: https://nightly.link/scikit-learn/scikit-learn/actions/runs/${{ github.run_id }} + GITHUB_RUN_URL: https://nightly.link/${{ github.github_repository }}/actions/runs/${{ github.run_id }} CCI_TOKEN: ${{ secrets.CCI_TOKEN }} From a301800019b88a2abe83d551bd36c4b598246874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 23:14:10 +0200 Subject: [PATCH 54/90] Trigger build From e40b5470dbb6c3a236390f51ef2258d005be0581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 23:31:00 +0200 Subject: [PATCH 55/90] Trigger build --- .github/workflows/push-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 11c56209cfd23..3c044c9ae0bb2 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -19,5 +19,5 @@ jobs: - name: Trigger hosting jobs run: bash build_tools/github/trigger_hosting.sh env: - GITHUB_RUN_URL: https://nightly.link/${{ github.github_repository }}/actions/runs/${{ github.run_id }} + GITHUB_RUN_URL: https://nightly.link/${{ github.repository }}/actions/runs/${{ github.run_id }} CCI_TOKEN: ${{ secrets.CCI_TOKEN }} From 14076f5cfa3f47479037808826f81031032592b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Thu, 28 Apr 2022 23:32:56 +0200 Subject: [PATCH 56/90] Trigger build From 8b70d031c7aefbcb74e8096f3df70c7691d470eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 08:59:51 +0200 Subject: [PATCH 57/90] [FIX] Fix environment variables --- .circleci/config.yml | 3 ++- .github/workflows/push-docs.yml | 2 +- build_tools/github/trigger_hosting.sh | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1677648a4432c..bd30efba16c5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 -# Parameters required to conditionally execute the "deploy" job and +# Parameters required to trigger the execution +# of the "doc-min-dependencies" and "doc" jobs parameters: GITHUB_RUN_URL: type: string diff --git a/.github/workflows/push-docs.yml b/.github/workflows/push-docs.yml index 3c044c9ae0bb2..5b8d047afae0e 100644 --- a/.github/workflows/push-docs.yml +++ b/.github/workflows/push-docs.yml @@ -19,5 +19,5 @@ jobs: - name: Trigger hosting jobs run: bash build_tools/github/trigger_hosting.sh env: - GITHUB_RUN_URL: https://nightly.link/${{ github.repository }}/actions/runs/${{ github.run_id }} CCI_TOKEN: ${{ secrets.CCI_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index 81fa6b3880449..83e89d577715c 100644 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -4,9 +4,9 @@ set -e set -x curl --request POST \ - --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/continue \ + --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/pipeline \ --header "Circle-Token: $CCI_TOKEN" \ --header "content-type: application/json" \ --header "x-attribution-actor-id: github_actions" \ --header "x-attribution-login: github_actions" \ - --data \{\"parameters\":\{\"GITHUB_RUN_URL\":\"$GITHUB_RUN_URL\"\}\} + --data \{\"parameters\":\{\"GITHUB_RUN_URL\":\"https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID\"\}\} From dce3df6c8c3325868902799980b9a7b258bb63ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:03:24 +0200 Subject: [PATCH 58/90] Trigger build From 380676a2b4aec07a327b678715cd0c4a7e33f3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:06:51 +0200 Subject: [PATCH 59/90] [FIX] Fix download script --- build_tools/circle/download_documentation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh index 8c46a8c55bc08..5c84c5f17afef 100644 --- a/build_tools/circle/download_documentation.sh +++ b/build_tools/circle/download_documentation.sh @@ -4,4 +4,4 @@ set -e set -x wget $GHA_META -unzip doc.zip -d doc +unzip doc*.zip -d doc From 101b3c32fb91dd13d61ff0021d7615ea2d7c5788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:27:57 +0200 Subject: [PATCH 60/90] [FIX] Fix run step --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd30efba16c5f..45d618b24cde3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip steps: - checkout - - run: ./build_tools/circle/download_documentation.sh + - run: bash build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc @@ -27,7 +27,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip steps: - checkout - - run: ./build_tools/circle/download_documentation.sh + - run: bash build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc From b0974a4df2becf3f8f2d148fb9bc6d32bb7f3685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:29:38 +0200 Subject: [PATCH 61/90] Trigger build From 5d1e91f889daee16009cc3756700918448ec4e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:48:40 +0200 Subject: [PATCH 62/90] [FIX] Fix environment variables --- build_tools/circle/download_documentation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh index 5c84c5f17afef..165dffa447214 100644 --- a/build_tools/circle/download_documentation.sh +++ b/build_tools/circle/download_documentation.sh @@ -3,5 +3,5 @@ set -e set -x -wget $GHA_META +wget $GITHUB_ARTIFACT_URL unzip doc*.zip -d doc From fe8b5870cb73433849e710d906f52380bee8f9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 09:50:32 +0200 Subject: [PATCH 63/90] Trigger build From fca68a0fc96cfecf46270819840090bc8c1d4281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 10:25:46 +0200 Subject: [PATCH 64/90] [FIX] Fix path to built documentation --- build_tools/circle/download_documentation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh index 165dffa447214..0a3a2b97a4370 100644 --- a/build_tools/circle/download_documentation.sh +++ b/build_tools/circle/download_documentation.sh @@ -4,4 +4,4 @@ set -e set -x wget $GITHUB_ARTIFACT_URL -unzip doc*.zip -d doc +unzip doc*.zip -d doc/_build/html/stable From 69f0a42f6ceb835aabeec3a09a99771757d8fe72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 10:28:04 +0200 Subject: [PATCH 65/90] Trigger build From b933f72e676410465be0279277edb1b5ef9bb2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 10:52:18 +0200 Subject: [PATCH 66/90] [FIX] Create path before extracting --- build_tools/circle/download_documentation.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh index 0a3a2b97a4370..c2d6d09d0abb9 100644 --- a/build_tools/circle/download_documentation.sh +++ b/build_tools/circle/download_documentation.sh @@ -4,4 +4,5 @@ set -e set -x wget $GITHUB_ARTIFACT_URL +mkdir -p doc/_build/html/stable unzip doc*.zip -d doc/_build/html/stable From 7099f23678cceb78d09d6e461fd651362dccfea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 12:51:04 +0200 Subject: [PATCH 67/90] [MNT] Tryo to trigger to the proper branch --- .github/workflows/build-docs.yml | 16 ++++++++++++++++ .../{push-docs.yml => trigger-hosting.yml} | 0 build_tools/github/store_branch.sh | 6 ++++++ build_tools/github/trigger_hosting.sh | 7 ++++++- 4 files changed, 28 insertions(+), 1 deletion(-) rename .github/workflows/{push-docs.yml => trigger-hosting.yml} (100%) create mode 100644 build_tools/github/store_branch.sh diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 71ba7f7b6821c..87625300e00eb 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -87,3 +87,19 @@ jobs: with: name: doc path: doc/_build/html/stable + + # Store the branch reference that triggered the + # workflow run because we must specify the same + # branch when triggering the pipeline that will + # host the documentation in the current service + store-branch: + - name: Checkout scikit-learn + uses: actions/checkout@v2 + + - name: Store branch reference + run: build_tools/github/store_branch.sh + + - name: Upload branch reference + with: + name: branch + path: branch.txt diff --git a/.github/workflows/push-docs.yml b/.github/workflows/trigger-hosting.yml similarity index 100% rename from .github/workflows/push-docs.yml rename to .github/workflows/trigger-hosting.yml diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh new file mode 100644 index 0000000000000..305c377b65512 --- /dev/null +++ b/build_tools/github/store_branch.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e +set -x + +echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$//" > branch.txt diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index 83e89d577715c..252670b4e4704 100644 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -3,10 +3,15 @@ set -e set -x +GITHUB_RUN_URL=https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID +BRANCH=$( wget $GITHUB_RUN_URL/branch.zip && \ + unzip branch.zip > /dev/null && \ + cat branch.txt ) + curl --request POST \ --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/pipeline \ --header "Circle-Token: $CCI_TOKEN" \ --header "content-type: application/json" \ --header "x-attribution-actor-id: github_actions" \ --header "x-attribution-login: github_actions" \ - --data \{\"parameters\":\{\"GITHUB_RUN_URL\":\"https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID\"\}\} + --data \{\"branch\":\"$BRANCH\", \"parameters\":\{\"GITHUB_RUN_URL\": $GITHUB_RUN_URL\"\"\}\} From d0ccf900350792d85accb6e42f9d72e3bdc73323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 12:54:52 +0200 Subject: [PATCH 68/90] Trigger build From fed17c92de546a3f036f3de43e56bc9fb28af298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 12:56:00 +0200 Subject: [PATCH 69/90] [FIX] Fix yaml --- .github/workflows/build-docs.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 87625300e00eb..6912fce63f1b8 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -93,13 +93,15 @@ jobs: # branch when triggering the pipeline that will # host the documentation in the current service store-branch: - - name: Checkout scikit-learn - uses: actions/checkout@v2 + runs-on: ubuntu-latest + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v2 - - name: Store branch reference - run: build_tools/github/store_branch.sh + - name: Store branch reference + run: build_tools/github/store_branch.sh - - name: Upload branch reference - with: - name: branch - path: branch.txt + - name: Upload branch reference + with: + name: branch + path: branch.txt From 686e461da6a349249b212fb4f59315d7e26602e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 12:57:45 +0200 Subject: [PATCH 70/90] [FIX] Fix yaml --- .github/workflows/build-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 6912fce63f1b8..d6fc51ce37a1d 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -102,6 +102,7 @@ jobs: run: build_tools/github/store_branch.sh - name: Upload branch reference + uses: actions/upload-artifact@v2 with: name: branch path: branch.txt From 37f2cf18f58fabf3510edeb3aa076044f4e19eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 12:58:24 +0200 Subject: [PATCH 71/90] Trigger build From 167a950c7d0603cc72ce8199be3c0edfd396db27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 13:01:23 +0200 Subject: [PATCH 72/90] [MNT] Set execution permissions --- .circleci/config.yml | 4 ++-- build_tools/circle/download_documentation.sh | 0 build_tools/github/store_branch.sh | 0 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 build_tools/circle/download_documentation.sh mode change 100644 => 100755 build_tools/github/store_branch.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 45d618b24cde3..bd30efba16c5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip steps: - checkout - - run: bash build_tools/circle/download_documentation.sh + - run: ./build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc @@ -27,7 +27,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip steps: - checkout - - run: bash build_tools/circle/download_documentation.sh + - run: ./build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh old mode 100644 new mode 100755 diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh old mode 100644 new mode 100755 From 7c940a2f5cf192a25b2ca7ab9596de1da6afb955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 13:03:12 +0200 Subject: [PATCH 73/90] Trigger build From 70a6ae9247fbf7cb0fe1cbbdba21783e40f83dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 13:59:17 +0200 Subject: [PATCH 74/90] [FIX] Remove whitespaces --- build_tools/github/trigger_hosting.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index 252670b4e4704..bf7e4bb143c2d 100644 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -14,4 +14,4 @@ curl --request POST \ --header "content-type: application/json" \ --header "x-attribution-actor-id: github_actions" \ --header "x-attribution-login: github_actions" \ - --data \{\"branch\":\"$BRANCH\", \"parameters\":\{\"GITHUB_RUN_URL\": $GITHUB_RUN_URL\"\"\}\} + --data \{\"branch\":\"$BRANCH\",\"parameters\":\{\"GITHUB_RUN_URL\":$GITHUB_RUN_URL\"\"\}\} From b391bfafe21fe07c347a61fc1364ada7108fd539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 14:23:31 +0200 Subject: [PATCH 75/90] [FIX] Fix trailing quotes --- build_tools/github/trigger_hosting.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index bf7e4bb143c2d..8d0e16eee6275 100644 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -14,4 +14,4 @@ curl --request POST \ --header "content-type: application/json" \ --header "x-attribution-actor-id: github_actions" \ --header "x-attribution-login: github_actions" \ - --data \{\"branch\":\"$BRANCH\",\"parameters\":\{\"GITHUB_RUN_URL\":$GITHUB_RUN_URL\"\"\}\} + --data \{\"branch\":\"$BRANCH\",\"parameters\":\{\"GITHUB_RUN_URL\":\"$GITHUB_RUN_URL\"\}\} From 0f1d39ae9d186a66a4bdef5fd519570d62ac1b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 15:32:07 +0200 Subject: [PATCH 76/90] [Fix] Fix run step in jobs --- .github/workflows/build-docs.yml | 6 +++--- build_tools/github/trigger_hosting.sh | 0 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 build_tools/github/trigger_hosting.sh diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index d6fc51ce37a1d..8bb4d9b888343 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -26,7 +26,7 @@ jobs: uses: actions/setup-python@v2 - name: Build documentation - run: build_tools/github/build_doc.sh + run: bash build_tools/github/build_doc.sh env: OMP_NUM_THREADS: 2 MKL_NUM_THREADS: 2 @@ -63,7 +63,7 @@ jobs: uses: actions/setup-python@v2 - name: Build documentation - run: build_tools/github/build_doc.sh + run: bash build_tools/github/build_doc.sh env: OMP_NUM_THREADS: 2 MKL_NUM_THREADS: 2 @@ -99,7 +99,7 @@ jobs: uses: actions/checkout@v2 - name: Store branch reference - run: build_tools/github/store_branch.sh + run: bash build_tools/github/store_branch.sh - name: Upload branch reference uses: actions/upload-artifact@v2 diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh old mode 100644 new mode 100755 From 913a0712a237938b00693defdf23e7ce122f7017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 15:47:12 +0200 Subject: [PATCH 77/90] [Fix] Fix permissions --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd30efba16c5f..45d618b24cde3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip steps: - checkout - - run: ./build_tools/circle/download_documentation.sh + - run: bash build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc @@ -27,7 +27,7 @@ jobs: - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip steps: - checkout - - run: ./build_tools/circle/download_documentation.sh + - run: bash build_tools/circle/download_documentation.sh - store_artifacts: path: doc/_build/html/stable destination: doc From b016b07c438a1bb18f73336435cbfb11ef7fd20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 20:12:53 +0200 Subject: [PATCH 78/90] [MNT] Replace merge by head --- build_tools/github/store_branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh index 305c377b65512..2957c38b8b6f1 100755 --- a/build_tools/github/store_branch.sh +++ b/build_tools/github/store_branch.sh @@ -3,4 +3,4 @@ set -e set -x -echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$//" > branch.txt +echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$/\/head/" > branch.txt From 697f32f184f1e056ec4eb394c3dd5ed59b5cb869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 21:17:29 +0200 Subject: [PATCH 79/90] [DBG] Debug github event object --- .github/workflows/build-docs.yml | 2 ++ build_tools/github/store_branch.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8bb4d9b888343..30f8645f5e9f1 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -100,6 +100,8 @@ jobs: - name: Store branch reference run: bash build_tools/github/store_branch.sh + env: + EVENT_OBJ: ${{ toJson(github.event) }} - name: Upload branch reference uses: actions/upload-artifact@v2 diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh index 2957c38b8b6f1..24b57fb52d4a2 100755 --- a/build_tools/github/store_branch.sh +++ b/build_tools/github/store_branch.sh @@ -3,4 +3,6 @@ set -e set -x +echo $EVENT_OBJ + echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$/\/head/" > branch.txt From 5dc97792413759968d422f58e35f605dc7684c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 21:19:16 +0200 Subject: [PATCH 80/90] Trigger build [doc quick] From a4fbbe0ac5b1b9a05c668dff786c5c2b8dd24de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 21:23:49 +0200 Subject: [PATCH 81/90] [DBG] Debug github event object --- .github/workflows/build-docs.yml | 2 -- .github/workflows/trigger-hosting.yml | 1 + build_tools/github/store_branch.sh | 2 -- build_tools/github/trigger_hosting.sh | 2 ++ 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 30f8645f5e9f1..8bb4d9b888343 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -100,8 +100,6 @@ jobs: - name: Store branch reference run: bash build_tools/github/store_branch.sh - env: - EVENT_OBJ: ${{ toJson(github.event) }} - name: Upload branch reference uses: actions/upload-artifact@v2 diff --git a/.github/workflows/trigger-hosting.yml b/.github/workflows/trigger-hosting.yml index 5b8d047afae0e..62f73da014c10 100644 --- a/.github/workflows/trigger-hosting.yml +++ b/.github/workflows/trigger-hosting.yml @@ -21,3 +21,4 @@ jobs: env: CCI_TOKEN: ${{ secrets.CCI_TOKEN }} RUN_ID: ${{ github.event.workflow_run.id }} + EVENT_OBJ: ${{ toJson(github.event) }} diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh index 24b57fb52d4a2..2957c38b8b6f1 100755 --- a/build_tools/github/store_branch.sh +++ b/build_tools/github/store_branch.sh @@ -3,6 +3,4 @@ set -e set -x -echo $EVENT_OBJ - echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$/\/head/" > branch.txt diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index 8d0e16eee6275..e746ad86ebd3e 100755 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -3,6 +3,8 @@ set -e set -x +echo $EVENT_OBJ + GITHUB_RUN_URL=https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID BRANCH=$( wget $GITHUB_RUN_URL/branch.zip && \ unzip branch.zip > /dev/null && \ From a735c838ae488615fe806bf550da69443918b304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Fri, 29 Apr 2022 21:26:57 +0200 Subject: [PATCH 82/90] Trigger build [doc quick] From 7e725b6bb06528621c90813914d84761dbedcb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Sun, 1 May 2022 11:57:00 +0200 Subject: [PATCH 83/90] Trigger build [doc quick] From 63cf94351719f620ca5a287e0bcf7f992bb6017d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Mon, 2 May 2022 14:01:05 +0200 Subject: [PATCH 84/90] [MNT] Simplify trigger hosting --- .github/workflows/build-docs.yml | 19 ------------------- .github/workflows/trigger-hosting.yml | 4 +++- build_tools/github/store_branch.sh | 6 ------ build_tools/github/trigger_hosting.sh | 12 +++++++----- 4 files changed, 10 insertions(+), 31 deletions(-) delete mode 100755 build_tools/github/store_branch.sh diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8bb4d9b888343..9df2f84d0371a 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -87,22 +87,3 @@ jobs: with: name: doc path: doc/_build/html/stable - - # Store the branch reference that triggered the - # workflow run because we must specify the same - # branch when triggering the pipeline that will - # host the documentation in the current service - store-branch: - runs-on: ubuntu-latest - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v2 - - - name: Store branch reference - run: bash build_tools/github/store_branch.sh - - - name: Upload branch reference - uses: actions/upload-artifact@v2 - with: - name: branch - path: branch.txt diff --git a/.github/workflows/trigger-hosting.yml b/.github/workflows/trigger-hosting.yml index 62f73da014c10..40a558b9f5cc7 100644 --- a/.github/workflows/trigger-hosting.yml +++ b/.github/workflows/trigger-hosting.yml @@ -20,5 +20,7 @@ jobs: run: bash build_tools/github/trigger_hosting.sh env: CCI_TOKEN: ${{ secrets.CCI_TOKEN }} + EVENT: ${{ github.event.workflow_run.event }} RUN_ID: ${{ github.event.workflow_run.id }} - EVENT_OBJ: ${{ toJson(github.event) }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + PULL_REQUEST_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} diff --git a/build_tools/github/store_branch.sh b/build_tools/github/store_branch.sh deleted file mode 100755 index 2957c38b8b6f1..0000000000000 --- a/build_tools/github/store_branch.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e -set -x - -echo $GITHUB_REF | sed -e "s/^refs\/\(heads\/\)\{0,1\}//" -e "s/\/merge$/\/head/" > branch.txt diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index e746ad86ebd3e..4c71fcdd19aad 100755 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -3,12 +3,14 @@ set -e set -x -echo $EVENT_OBJ - GITHUB_RUN_URL=https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID -BRANCH=$( wget $GITHUB_RUN_URL/branch.zip && \ - unzip branch.zip > /dev/null && \ - cat branch.txt ) + +if [ "$EVENT" == pull_request ] +then + BRANCH=pull/$PULL_REQUEST_NUMBER/head +else + BRANCH=$HEAD_BRANCH +fi curl --request POST \ --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/pipeline \ From 94457f62cc2837070b205481eceeb098ff1ecfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Mon, 2 May 2022 14:02:32 +0200 Subject: [PATCH 85/90] Trigger build [doc quick] From 1dd57c33ef42f863fcc9a03d27e8e1ce7e5540b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Mon, 2 May 2022 15:48:25 +0200 Subject: [PATCH 86/90] [FIX] Fix double execution of jobs --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45d618b24cde3..0a22c23970ee2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,8 +90,7 @@ workflows: when: not: equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] - # The following jobs will be executed only if the - # action is executed to ignore normal push events + # The jobs should run only when triggered by the workflow jobs: - doc-min-dependencies - doc @@ -100,5 +99,9 @@ workflows: - doc linux-arm64: + when: + equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] + # Prevent double execution of this job: on push + # by default and when triggered by the workflow jobs: - linux-arm64 From 5e5699e8072cee4aa3f096d09fa502b38a3f14f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Mon, 2 May 2022 15:49:32 +0200 Subject: [PATCH 87/90] Trigger build [doc quick] From 13fe9789935865ecaaa9a7706a1cb19e27bdabc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Tue, 31 May 2022 17:15:55 +0200 Subject: [PATCH 88/90] [MNT] Remove unnecesary environment variables --- .github/workflows/build-docs.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index d5d71a3911523..286f7e16e2936 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -32,18 +32,6 @@ jobs: MKL_NUM_THREADS: 2 CONDA_ENV_NAME: testenv LOCK_FILE: build_tools/github/doc_min_dependencies_linux-64_conda.lock - PYTHON_VERSION: 3.8 - NUMPY_VERSION: min - SCIPY_VERSION: min - MATPLOTLIB_VERSION: min - CYTHON_VERSION: min - SCIKIT_IMAGE_VERSION: min - SPHINX_VERSION: min - PANDAS_VERSION: min - SPHINX_GALLERY_VERSION: min - NUMPYDOC_VERSION: min - SPHINX_PROMPT_VERSION: min - SPHINXEXT_OPENGRAPH_VERSION: min - name: Upload documentation uses: actions/upload-artifact@v2 From f8a0fa10ddce139fb2ad840d9c23ef11d565e298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Tue, 31 May 2022 17:30:51 +0200 Subject: [PATCH 89/90] Trigger build [doc quick] From 0929e742a2171d9fb669abc4252b28de0546d590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Carlos=20Alfaro=20Jim=C3=A9nez?= Date: Tue, 31 May 2022 18:28:33 +0200 Subject: [PATCH 90/90] [MNT] Use CIRCLE_CI_TOKEN for the environment variable --- .github/workflows/trigger-hosting.yml | 2 +- build_tools/github/trigger_hosting.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trigger-hosting.yml b/.github/workflows/trigger-hosting.yml index a68f4f01433b1..417c1816f595c 100644 --- a/.github/workflows/trigger-hosting.yml +++ b/.github/workflows/trigger-hosting.yml @@ -19,7 +19,7 @@ jobs: - name: Trigger hosting jobs run: bash build_tools/github/trigger_hosting.sh env: - CCI_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} + CIRCLE_CI_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} EVENT: ${{ github.event.workflow_run.event }} RUN_ID: ${{ github.event.workflow_run.id }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh index 4c71fcdd19aad..4fc9b0dccbd54 100755 --- a/build_tools/github/trigger_hosting.sh +++ b/build_tools/github/trigger_hosting.sh @@ -14,7 +14,7 @@ fi curl --request POST \ --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/pipeline \ - --header "Circle-Token: $CCI_TOKEN" \ + --header "Circle-Token: $CIRCLE_CI_TOKEN" \ --header "content-type: application/json" \ --header "x-attribution-actor-id: github_actions" \ --header "x-attribution-login: github_actions" \