From 26877127729cb9b6742f62b174c3e80c0d6fed6c Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 14 Dec 2018 14:40:11 +0800 Subject: [PATCH 1/4] ci: fix checking for presence of CI_GITHUB_API_KEY CI_GITHUB_API_KEY is always set when evaluating job include/exclude conditions, and it only gets removed when the job starts. Move the check into the script. --- .travis.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 276b393141..5337825488 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,22 +63,25 @@ jobs: - BUILD_TYPE=style_check install: - > - [ "$BUILD_TYPE" = "style_check" ] && { - [ -f $HOME/astyle/build/gcc/bin/astyle ] || { - wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download; - tar -xf astyle_3.1_linux.tar.gz -C $HOME; - make -C $HOME/astyle/build/gcc; - } - make -C $HOME/astyle/build/gcc prefix=$HOME install; - } || true + if [ ! -f $HOME/astyle/build/gcc/bin/astyle ]; then + wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download; + tar -xf astyle_3.1_linux.tar.gz -C $HOME; + make -C $HOME/astyle/build/gcc; + fi + make -C $HOME/astyle/build/gcc prefix=$HOME install; # Deploy stage. # Here we build the package JSON (always) and do the deployments - name: "Package / deploy" stage: deploy - script: $TRAVIS_BUILD_DIR/tests/common.sh + script: + - > + if [ -n "$CI_GITHUB_API_KEY" ]; then + $TRAVIS_BUILD_DIR/tests/common.sh + else + echo "Not building the package." + fi env: BUILD_TYPE=package - if: env(CI_GITHUB_API_KEY) IS present deploy: # Create Github release, upload artifacts - provider: releases From 0dd5f034f2415bc273945ad7d80cde1e30e5d93c Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 14 Dec 2018 16:53:54 +0800 Subject: [PATCH 2/4] ci: split build jobs into separate shell scripts --- .travis.yml | 45 ++++++++------------------ tests/ci/build_boards.sh | 14 ++++++++ tests/ci/build_docs.sh | 9 ++++++ tests/ci/build_package.sh | 16 +++++++++ tests/ci/host_test.sh | 10 ++++++ tests/ci/install_astyle.sh | 16 +++++++++ tests/ci/style_check.sh | 16 +++++++++ tests/common.sh | 66 +------------------------------------- 8 files changed, 95 insertions(+), 97 deletions(-) create mode 100755 tests/ci/build_boards.sh create mode 100755 tests/ci/build_docs.sh create mode 100755 tests/ci/build_package.sh create mode 100755 tests/ci/host_test.sh create mode 100755 tests/ci/install_astyle.sh create mode 100755 tests/ci/style_check.sh diff --git a/.travis.yml b/.travis.yml index 5337825488..bc64065d85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,18 +13,14 @@ stages: jobs: include: # Build stage. To save time, run all kinds of builds and tests in parallel. - # - # TODO: since we can now call different script for each job, - # split the do-it-all common.sh into separate scripts responsible - # for different types of jobs below: - name: "Host tests" stage: build - script: $TRAVIS_BUILD_DIR/tests/common.sh - env: - - BUILD_TYPE=host_tests - install: - - sudo apt-get install valgrind lcov + script: $TRAVIS_BUILD_DIR/tests/ci/host_test.sh + install: sudo apt-get install valgrind lcov + # TODO: since we can now call different script for each job, + # split the do-it-all common.sh into separate scripts responsible + # for different types of jobs below: - name: "Build (1)" script: $TRAVIS_BUILD_DIR/tests/common.sh env: @@ -51,36 +47,21 @@ jobs: - BUILD_TYPE=platformio_odd - name: "Docs" - script: $TRAVIS_BUILD_DIR/tests/common.sh - env: - - BUILD_TYPE=docs - install: - - pip install --user -r doc/requirements.txt; + script: $TRAVIS_BUILD_DIR/tests/ci/build_docs.sh + install: pip install --user -r doc/requirements.txt; - name: "Style check" - script: $TRAVIS_BUILD_DIR/tests/common.sh - env: - - BUILD_TYPE=style_check - install: - - > - if [ ! -f $HOME/astyle/build/gcc/bin/astyle ]; then - wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download; - tar -xf astyle_3.1_linux.tar.gz -C $HOME; - make -C $HOME/astyle/build/gcc; - fi - make -C $HOME/astyle/build/gcc prefix=$HOME install; + script: $TRAVIS_BUILD_DIR/tests/ci/style_check.sh + install: tests/ci/install_astyle.sh + + - name: "Boards" + script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh # Deploy stage. # Here we build the package JSON (always) and do the deployments - name: "Package / deploy" stage: deploy - script: - - > - if [ -n "$CI_GITHUB_API_KEY" ]; then - $TRAVIS_BUILD_DIR/tests/common.sh - else - echo "Not building the package." - fi + script: tests/ci/build_package.sh env: BUILD_TYPE=package deploy: # Create Github release, upload artifacts diff --git a/tests/ci/build_boards.sh b/tests/ci/build_boards.sh new file mode 100755 index 0000000000..a263bdad28 --- /dev/null +++ b/tests/ci/build_boards.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# CI job which checks that boards.txt and package_esp8266com_index.template.json are up to date + +set -ev + +cd $TRAVIS_BUILD_DIR + +tools/boards.txt.py --boardsgen --ldgen --packagegen --docgen + +git diff --exit-code -- boards.txt \ + doc/boards.rst \ + tools/sdk/ld/ +git diff --exit-code -w -- package/package_esp8266com_index.template.json diff --git a/tests/ci/build_docs.sh b/tests/ci/build_docs.sh new file mode 100755 index 0000000000..31db9f8f9f --- /dev/null +++ b/tests/ci/build_docs.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# CI job to run the documentation build + +set -ev + +cd $TRAVIS_BUILD_DIR/doc + +SPHINXOPTS="-W" make html diff --git a/tests/ci/build_package.sh b/tests/ci/build_package.sh new file mode 100755 index 0000000000..11e9acf6b0 --- /dev/null +++ b/tests/ci/build_package.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# CI job which builds the boards manager package + +set -ev + +export PKG_URL=https://github.com/esp8266/Arduino/releases/download/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip +export DOC_URL=https://arduino-esp8266.readthedocs.io/en/$TRAVIS_TAG/ + +if [ -z "$CI_GITHUB_API_KEY" ]; then + echo "Github API key not set. Skip building the package." + exit 0 +fi + +cd $TRAVIS_BUILD_DIR/package +./build_boards_manager_package.sh diff --git a/tests/ci/host_test.sh b/tests/ci/host_test.sh new file mode 100755 index 0000000000..e20e23b331 --- /dev/null +++ b/tests/ci/host_test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# +# CI job for running tests on the host + +set -ev + +cd $TRAVIS_BUILD_DIR/tests/host + +make CI +make clean-objects diff --git a/tests/ci/install_astyle.sh b/tests/ci/install_astyle.sh new file mode 100755 index 0000000000..32c2f5efad --- /dev/null +++ b/tests/ci/install_astyle.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# $HOME/astyle directory is cached on Travis. +# If cached build is not present, download astyle and build it. +# Install built astyle binary into the home directory. +# + +set -e + +if [ ! -f $HOME/astyle/build/gcc/bin/astyle ]; then + wget -O astyle_3.1_linux.tar.gz https://sourceforge.net/projects/astyle/files/astyle/astyle%203.1/astyle_3.1_linux.tar.gz/download + tar -xf astyle_3.1_linux.tar.gz -C $HOME + make -C $HOME/astyle/build/gcc +fi + +make -C $HOME/astyle/build/gcc prefix=$HOME install diff --git a/tests/ci/style_check.sh b/tests/ci/style_check.sh new file mode 100755 index 0000000000..d51d9c54cd --- /dev/null +++ b/tests/ci/style_check.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# CI job for checking examples style + +set -ev + +find $TRAVIS_BUILD_DIR/libraries -name '*.ino' -exec \ + astyle \ + --suffix=none \ + --options=$TRAVIS_BUILD_DIR/tests/examples_style.conf {} \; + +# Revert changes which astyle might have done to the submodules, +# as we don't want to fail the build because of the 3rd party libraries +git submodule foreach --recursive git reset --hard + +git diff --exit-code -- $TRAVIS_BUILD_DIR/libraries diff --git a/tests/common.sh b/tests/common.sh index e9fa151458..c62369fa38 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -126,37 +126,6 @@ function install_ide() export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" } -function build_docs() -{ - SPHINXOPTS="-W" make html -} - -function run_host_tests() -{ - pushd host - make CI - make clean-objects - popd -} - -function build_package() -{ - export PKG_URL=https://github.com/esp8266/Arduino/releases/download/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip - export DOC_URL=https://arduino-esp8266.readthedocs.io/en/$TRAVIS_TAG/ - ./build_boards_manager_package.sh -} - -function build_boards() -{ - echo -e "travis_fold:start:build_boards" - tools/boards.txt.py --boardsgen --ldgen --packagegen --docgen - git diff --exit-code -- boards.txt \ - doc/boards.rst \ - tools/sdk/ld/ - git diff --exit-code -w -- package/package_esp8266com_index.template.json - echo -e "travis_fold:end:build_boards" -} - function install_platformio() { pip install --user -U https://github.com/platformio/platformio/archive/develop.zip @@ -239,22 +208,6 @@ function build_sketches_with_arduino() echo -e "travis_fold:end:size_report" } -function check_examples_style() -{ - echo -e "travis_fold:start:check_examples_style" - - find $TRAVIS_BUILD_DIR/libraries -name '*.ino' -exec \ - astyle \ - --suffix=none \ - --options=$TRAVIS_BUILD_DIR/tests/examples_style.conf {} \; - - # we have no control over submodules - git submodule foreach --recursive git reset --hard - - git diff --exit-code -- $TRAVIS_BUILD_DIR/libraries - - echo -e "travis_fold:end:check_examples_style" -} set -e @@ -293,25 +246,8 @@ elif [ "$BUILD_TYPE" = "platformio_odd" ]; then # PlatformIO install_platformio build_sketches_with_platformio $TRAVIS_BUILD_DIR/libraries "--board nodemcuv2 --verbose" 2 1 -elif [ "$BUILD_TYPE" = "docs" ]; then - # Build documentation using Sphinx - cd $TRAVIS_BUILD_DIR/doc - build_docs -elif [ "$BUILD_TYPE" = "package" ]; then - # Check that boards.txt, ld scripts, package JSON template, and boards.rst are up to date - build_boards - # Build release package - cd $TRAVIS_BUILD_DIR/package - build_package -elif [ "$BUILD_TYPE" = "host_tests" ]; then - # Run host side tests - cd $TRAVIS_BUILD_DIR/tests - run_host_tests -elif [ "$BUILD_TYPE" = "style_check" ]; then - # Check code style - check_examples_style else - echo "BUILD_TYPE not set" + echo "BUILD_TYPE not set or invalid" exit 1 fi From 351be9315e279afb0d3fc1b36e0a69639960775e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Dec 2018 02:04:49 +0100 Subject: [PATCH 3/4] Update .travis.yml use pip3 for doc generation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc64065d85..edc66fd902 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ jobs: - name: "Docs" script: $TRAVIS_BUILD_DIR/tests/ci/build_docs.sh - install: pip install --user -r doc/requirements.txt; + install: pip3 install --user -r doc/requirements.txt; - name: "Style check" script: $TRAVIS_BUILD_DIR/tests/ci/style_check.sh From 0206c20fb2c1792c5996e2bad5783be7c19dc0fd Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Dec 2018 02:14:54 +0100 Subject: [PATCH 4/4] Update .travis.yml install pip3 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index edc66fd902..0b1a6d65fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,9 @@ jobs: - name: "Docs" script: $TRAVIS_BUILD_DIR/tests/ci/build_docs.sh - install: pip3 install --user -r doc/requirements.txt; + install: + - sudo apt-get install python3-pip + - pip3 install --user -r doc/requirements.txt; - name: "Style check" script: $TRAVIS_BUILD_DIR/tests/ci/style_check.sh