From 4169f88cc135aa56d369c2429d31317a928341d2 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 11 Dec 2018 18:37:24 +0800 Subject: [PATCH 01/10] ci: use travis jobs to structure the CI flow This mostly re-phrases the existing flow using the new language. Ref. https://blog.travis-ci.com/2017-05-11-introducing-build-stages --- .travis.yml | 154 ++++++++++++++---------- package/build_boards_manager_package.sh | 10 +- 2 files changed, 99 insertions(+), 65 deletions(-) diff --git a/.travis.yml b/.travis.yml index 732b21b126..276b393141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,73 +6,103 @@ cache: directories: - $HOME/astyle -matrix: +stages: + - build + - deploy + +jobs: include: - - env: - - BUILD_TYPE=package - - env: - - BUILD_TYPE=build_even - - env: - - BUILD_TYPE=build_odd - - env: - - BUILD_TYPE=debug_even - - env: - - BUILD_TYPE=debug_odd - - env: - - BUILD_TYPE=platformio_even - - env: - - BUILD_TYPE=platformio_odd - - env: - - BUILD_TYPE=docs - - env: - - BUILD_TYPE=host_tests - - env: - - BUILD_TYPE=style_check + # Build stage. To save time, run all kinds of builds and tests in parallel. + # + # TODO: since we can now call different script for each job, + # split the do-it-all common.sh into separate scripts responsible + # for different types of jobs below: + - name: "Host tests" + stage: build + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=host_tests + install: + - sudo apt-get install valgrind lcov + + - name: "Build (1)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=build_even + - name: "Build (2)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=build_odd + - name: "Debug (1)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=debug_even + - name: "Debug (2)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=debug_odd + - name: "Platformio (1)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=platformio_even + - name: "Platformio (2)" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=platformio_odd -install: - - > - [ "$BUILD_TYPE" = "docs" ] && { - pip install --user -r doc/requirements.txt; - } || true - - > - [ "$BUILD_TYPE" = "style_check" ] && { - [ -f $HOME/astyle/build/gcc/bin/astyle ] || { - wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download; - tar -xf astyle_3.1_linux.tar.gz -C $HOME; - make -C $HOME/astyle/build/gcc; - } - make -C $HOME/astyle/build/gcc prefix=$HOME install; - } || true - - sudo apt-get install valgrind lcov + - name: "Docs" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=docs + install: + - pip install --user -r doc/requirements.txt; -script: - - $TRAVIS_BUILD_DIR/tests/common.sh + - name: "Style check" + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: + - BUILD_TYPE=style_check + install: + - > + [ "$BUILD_TYPE" = "style_check" ] && { + [ -f $HOME/astyle/build/gcc/bin/astyle ] || { + wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download; + tar -xf astyle_3.1_linux.tar.gz -C $HOME; + make -C $HOME/astyle/build/gcc; + } + make -C $HOME/astyle/build/gcc prefix=$HOME install; + } || true -deploy: - - provider: releases - draft: true - skip_cleanup: true - api_key: - secure: A4FBmqyhlzy33oPeZVolg2Q/A3ZcJ3WnRQqQJ3NAPy+qGM5xcboOYtwcLL9vKaHZGfUB7lUP9QVZFGou1Wrmo9DnPvAoe3+XvCaDRGzVMxeIpu7UStbBD4Knbh98tlbMvZCXYRlT4VcusI9bMLK6UWw4sMdPislBh2FEfglTiag= - file_glob: true - tag_name: $TRAVIS_TAG - target_commitish: $TRAVIS_COMMIT - file: - - package/versions/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip - - package/versions/$TRAVIS_TAG/package_esp8266com_index.json - on: - repo: esp8266/Arduino - tags: true - condition: "$BUILD_TYPE = package" - - - provider: script - skip_cleanup: true - script: bash package/deploy_package_index.sh - on: - repo: esp8266/Arduino - tags: true - condition: "$BUILD_TYPE = package" + # Deploy stage. + # Here we build the package JSON (always) and do the deployments + - name: "Package / deploy" + stage: deploy + script: $TRAVIS_BUILD_DIR/tests/common.sh + env: BUILD_TYPE=package + if: env(CI_GITHUB_API_KEY) IS present + deploy: + # Create Github release, upload artifacts + - provider: releases + draft: true + skip_cleanup: true + api_key: + secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= + file_glob: true + tag_name: $TRAVIS_TAG + target_commitish: $TRAVIS_COMMIT + file: + - package/versions/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip + - package/versions/$TRAVIS_TAG/package_esp8266com_index.json + on: + repo: esp8266/Arduino + tags: true + # Update the package index URL to point to the new version + - provider: script + skip_cleanup: true + script: bash package/deploy_package_index.sh + on: + repo: esp8266/Arduino + tags: true notifications: email: diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 9e4468c58b..3f108d16d5 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -18,8 +18,7 @@ else plain_ver=$ver fi -# 'set -e' breaks CI but not local tests -#set -e +set -e package_name=esp8266-$ver echo "Version: $ver" @@ -134,8 +133,13 @@ fi cat $srcdir/package/package_esp8266com_index.template.json | \ jq "$jq_arg" > package_esp8266com_index.json +# Use Github API token, if available +curl_gh_token_arg=() +if [ ! -z "$CI_GITHUB_API_KEY" ]; then + curl_gh_token_arg=(-H "Authorization: token $CI_GITHUB_API_KEY") +fi # Get previous release name -curl --silent https://api.github.com/repos/esp8266/Arduino/releases > releases.json +curl --silent "${curl_gh_token_arg[@]}" https://api.github.com/repos/esp8266/Arduino/releases > releases.json # Previous final release (prerelase == false) prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' releases.json) # Previous release (possibly a pre-release) From 2c453923433cb817c9283b10ee28998556823960 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 16:51:50 +0100 Subject: [PATCH 02/10] igrr fix from feature/travis_jobs_tmp --- package/build_boards_manager_package.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 3f108d16d5..a5a5c0de5d 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -134,12 +134,18 @@ cat $srcdir/package/package_esp8266com_index.template.json | \ jq "$jq_arg" > package_esp8266com_index.json # Use Github API token, if available -curl_gh_token_arg=() +curl_gh_token_arg="" if [ ! -z "$CI_GITHUB_API_KEY" ]; then - curl_gh_token_arg=(-H "Authorization: token $CI_GITHUB_API_KEY") + echo "Authorization using CI_GITHUB_API_KEY" + curl_gh_token_arg="-H \"Authorization: token $CI_GITHUB_API_KEY\"" +else + echo "Warning: CI_GITHUB_API_KEY not set" fi +echo "====================================" +curl $curl_gh_token_arg -D - https://api.github.com/repos/esp8266/Arduino/releases +echo "====================================" # Get previous release name -curl --silent "${curl_gh_token_arg[@]}" https://api.github.com/repos/esp8266/Arduino/releases > releases.json +curl --silent $curl_gh_token_arg https://api.github.com/repos/esp8266/Arduino/releases > releases.json # Previous final release (prerelase == false) prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' releases.json) # Previous release (possibly a pre-release) From 75a83f6e54d55808aec6ab6da8d7bbabcd972135 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 16:53:11 +0100 Subject: [PATCH 03/10] git: depth 1, submodules not recursive --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 276b393141..34810ec719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,13 @@ cache: directories: - $HOME/astyle +git: + depth: 1 + submodules: false + +before_install: + - git submodule update --init + stages: - build - deploy From 98ee330317e8f265fba50805532475162f89f1e1 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 18:09:34 +0100 Subject: [PATCH 04/10] wip --- .travis.yml | 2 +- package/build_boards_manager_package.sh | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 34810ec719..8bdd088f0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - git submodule update --init stages: - - build +# - build - deploy jobs: diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index a5a5c0de5d..702544ed85 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -49,13 +49,14 @@ rm -rf package/versions/$ver mkdir -p $outdir # Get submodules -modules=libraries/SoftwareSerial -for mod in $modules; do - echo "refreshing submodule: $mod" - git submodule update --init -- $mod - (cd $mod && git reset --hard) -done -echo "done with submodules" +# useless: already cloned by travis +#modules=libraries/SoftwareSerial +#for mod in $modules; do +# echo "refreshing submodule: $mod" +# git submodule update --init -- $mod +# (cd $mod && git reset --hard) +#done +#echo "done with submodules" # Some files should be excluded from the package cat << EOF > exclude.txt @@ -141,11 +142,12 @@ if [ ! -z "$CI_GITHUB_API_KEY" ]; then else echo "Warning: CI_GITHUB_API_KEY not set" fi -echo "====================================" -curl $curl_gh_token_arg -D - https://api.github.com/repos/esp8266/Arduino/releases -echo "====================================" +#echo "====================================" +#curl $curl_gh_token_arg -D - https://api.github.com/repos/esp8266/Arduino/releases +#echo "====================================" # Get previous release name -curl --silent $curl_gh_token_arg https://api.github.com/repos/esp8266/Arduino/releases > releases.json +echo "debug: curl --silent -D /dev/stderr $curl_gh_token_arg https://api.github.com/repos/esp8266/Arduino/releases" +curl --silent -D /dev/stderr $curl_gh_token_arg https://api.github.com/repos/esp8266/Arduino/releases > releases.json # Previous final release (prerelase == false) prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' releases.json) # Previous release (possibly a pre-release) From b557c594ea65b6e3930a7e547ace2a771c92ce0e Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 19:04:26 +0100 Subject: [PATCH 05/10] wip --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8bdd088f0c..73976d22ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,10 @@ stages: # - build - deploy +env: + global: + - secure: "vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" + jobs: include: # Build stage. To save time, run all kinds of builds and tests in parallel. @@ -91,8 +95,8 @@ jobs: - provider: releases draft: true skip_cleanup: true - api_key: - secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= + #api_key: + # secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= file_glob: true tag_name: $TRAVIS_TAG target_commitish: $TRAVIS_COMMIT From 6a3399712902e4a18c4dd702397dd27bf9e44d23 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 19:15:08 +0100 Subject: [PATCH 06/10] wip --- .travis.yml | 4 ---- package/build_boards_manager_package.sh | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 73976d22ff..bdecff2a83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,6 @@ stages: # - build - deploy -env: - global: - - secure: "vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" - jobs: include: # Build stage. To save time, run all kinds of builds and tests in parallel. diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 702544ed85..c8f950a841 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -3,6 +3,8 @@ #set -x +CI_GITHUB_API_KEY="vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" + # Extract next version from platform.txt next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt` From 26115a11805cab14290c886ccc6c2953241899f4 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 19:24:07 +0100 Subject: [PATCH 07/10] restore full build, comment, temporary state --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bdecff2a83..5fd6edd5ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - git submodule update --init stages: -# - build + - build - deploy jobs: @@ -91,7 +91,7 @@ jobs: - provider: releases draft: true skip_cleanup: true - #api_key: + #api_key: -CURRENTLY HARDCODED IN package/deploy_package_index.sh # secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= file_glob: true tag_name: $TRAVIS_TAG From 892ca4fca21c515ddbbe61c8299d8b20a9c0c45a Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 20:08:57 +0100 Subject: [PATCH 08/10] wip --- .travis.yml | 4 ++-- package/build_boards_manager_package.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fd6edd5ad..86dfec11a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,8 +91,8 @@ jobs: - provider: releases draft: true skip_cleanup: true - #api_key: -CURRENTLY HARDCODED IN package/deploy_package_index.sh - # secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= + api_key: + secure: vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ= file_glob: true tag_name: $TRAVIS_TAG target_commitish: $TRAVIS_COMMIT diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index c8f950a841..eb90fe7b40 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -3,7 +3,7 @@ #set -x -CI_GITHUB_API_KEY="vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" +#CI_GITHUB_API_KEY="vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" # Extract next version from platform.txt next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt` From 84a0f8df9672a52a33927e315fbf9e15334add3e Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 20:10:57 +0100 Subject: [PATCH 09/10] wip --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 86dfec11a2..e1ab61527e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - git submodule update --init stages: - - build +# - build - deploy jobs: From 680efea8c39c061578f4e19bdfecec6d4574d380 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Tue, 11 Dec 2018 21:03:24 +0100 Subject: [PATCH 10/10] cleaning --- package/build_boards_manager_package.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index eb90fe7b40..3500b1ee5e 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -1,10 +1,6 @@ #!/bin/bash # -#set -x - -#CI_GITHUB_API_KEY="vt6StBC+ghqnh8YrreNo3wAzGGddJ2S4YpVZkz4S84xLEGWkIEghQhTrjlhzjBsrnHfLNko4tz9EsNx0yQ8yBlPOdReETGAkqnAU7PSPFss0qGcCRUXYtozNjbFQq6TWIxECK4xq40R9tE6NyeOpm9AYJtwF/v18u2T+T9qSgGQ=" - # Extract next version from platform.txt next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt`