From fb50bee1b2e91ed64b90f13a6bbb7d98a30e1ef7 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:28:02 -0700 Subject: [PATCH 01/13] Pin Bazel versions --- .bazelrc | 4 ++ .travis.yml | 80 +++++++++++++++++++-------------------- tools/bazel | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 42 deletions(-) create mode 100644 .bazelrc create mode 100755 tools/bazel diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000..1fa20d447 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,4 @@ +common:v0.28 --config=noop + +# pick something trivial as a noop +common:noop --logging=3 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 5a42a0e2f..45fdd0682 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,59 +1,55 @@ -# trusty beta image has jdk8, gcc4.8.4 -dist: trusty +dist: xenial sudo: required -# xcode8 has jdk8 -osx_image: xcode8 +language: generic +osx_image: xcode10.1 + # Not technically required but suppresses 'Ruby' in Job status message. language: sh cache: - directories: - - .bazel_cache + directories: + - ~/.bazel_binaries + - .bazel_cache os: - linux - osx - windows -env: - # Linting is broken. Disable until fixed. - # See https://github.com/bazelbuild/rules_scala/pull/622 - # we want to test the last release - #- V=0.16.1 TEST_SCRIPT=test_lint.sh - - V=0.28.0 TEST_SCRIPT=test_rules_scala - #- V=0.14.1 TEST_SCRIPT=test_intellij_aspect.sh - - V=0.28.0 TEST_SCRIPT=test_reproducibility +_linux: &linux + os: linux +_osx: &osx + os: osx + osx_image: xcode10.1 +_windows: &windows + os: windows + before_install: + - | + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + choco install jdk8 -params 'installdir=c:\\java8' + choco install bazel --version ${V} + fi +jobs: + include: + # test_rules_scala + - stage: test_rules_scala + <<: *linux + env: TEST_SCRIPT=test_rules_scala + - <<: *osx + env: TEST_SCRIPT=test_rules_scala + - <<: *windows + env: TEST_SCRIPT=test_rules_scala + # test_reproducibility + - stage: test_reproducibility + <<: *linux + env: TEST_SCRIPT=test_reproducibility + - <<: *osx + env: TEST_SCRIPT=test_reproducibility + - <<: *windows + env: TEST_SCRIPT=test_reproducibility before_install: - - | - if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then - choco install jdk8 -params 'installdir=c:\\java8' - choco install bazel --version ${V} - else - if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then - OS=darwin - else - sudo sysctl kernel.unprivileged_userns_clone=1 - sudo add-apt-repository -y ppa:openjdk-r/ppa - sudo apt-get update -q - sudo apt-get install openjdk-8-jdk -y - sudo apt-get install libxml2-utils -y - OS=linux - fi - - if [[ $V =~ .*rc[0-9]+.* ]]; then - PRE_RC=$(expr "$V" : '\([0-9.]*\)rc.*') - RC_PRC=$(expr "$V" : '[0-9.]*\(rc.*\)') - URL="https://storage.googleapis.com/bazel/${PRE_RC}/${RC_PRC}/bazel-${V}-installer-${OS}-x86_64.sh" - else - URL="https://github.com/bazelbuild/bazel/releases/download/${V}/bazel-${V}-installer-${OS}-x86_64.sh" - fi - wget -O install.sh "${URL}" - chmod +x install.sh - ./install.sh --user - rm -f install.sh - fi - cat .bazelrc.travis >> .bazelrc script: diff --git a/tools/bazel b/tools/bazel new file mode 100755 index 000000000..02198511d --- /dev/null +++ b/tools/bazel @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -e + +workspace="$(cd "$(dirname "$0")"/..; pwd)" + +default_bazel_version='0.28.1' + +case $(uname -s) in + Darwin|Linux) + # shellcheck disable=SC2153 + if [ -z "$BAZEL_VERSION" ]; then + bazel_version="$default_bazel_version" + else + bazel_version="$BAZEL_VERSION" + fi + ;; + *) + # windows, presumably + bazel_version='host' + ;; +esac + +case "$bazel_version" in + 'host') + bazel_version=$("$BAZEL_REAL" version | awk '/Build label/ {print $3}' | cut -d '-' -f 1) + bazel="$BAZEL_REAL" + ;; + '0.28.1') + darwin_sha='5d50ae13ba01a224ddf54cfd818289bee5b38e551cca22bffc79b89f377d2095' + linux_sha='a2a7e4cb38d7bc774a5bbfab4d45293f5f2158eb6caf0128792cc1148732a4e6' + ;; + *) + echo "The requested Bazel version '$bazel_version' is not supported" + exit 1 + ;; +esac + +if [ -z "$bazel" ]; then + bazel_bin_loc=~/.bazel_binaries + bazel=$bazel_bin_loc/$bazel_version/bin/bazel-real + + if ! [ -f "$bazel" ]; then + case $(uname -s) in + Darwin) + platform='darwin' + sha=$darwin_sha + ;; + Linux) + platform='linux' + sha=$linux_sha + ;; + *) + echo 'Your OS is not supported for automatic bazel installs.' + exit 1 + ;; + esac + remote_source=https://github.com/bazelbuild/bazel/releases/download + installer_name="bazel-$bazel_version-installer-$platform-x86_64.sh" + url="$remote_source/$bazel_version/$installer_name" + ( + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf $tmp_dir" EXIT + cd "$tmp_dir" + (>&2 echo "downloading installer from") + (>&2 echo "$url") + curl -o installer.sh -L "$url" + generated_sha=$(shasum -a 256 installer.sh | awk '{print $1}') + if [ "$generated_sha" != "$sha" ]; then + echo "Sha 256 does not match, expected: $sha" + echo "But found $generated_sha" + echo "Recommend you: update the sha to the expected" + echo "and then re-run this script" + exit 1 + fi + mkdir -p $bazel_bin_loc + chmod +x installer.sh + ./installer.sh --base="$bazel_bin_loc"/"$bazel_version" --bin="$bazel_bin_loc"/"$bazel_version"/bin_t + ) >&2 + fi +fi + +extra_command_args=() + +# shellcheck disable=SC2034 +IFS=. read -r major minor patch < <(echo "$bazel_version") +extra_command_args+=("--config=v$major.$minor") + +for (( i=1; i<=$#; i++ )) +do + case "${!i}" in + -*) + ;; + *) + n=$((i + 1)) + set -- "${@:1:$i}" "${extra_command_args[@]}" "${@:$n}" + break + ;; + esac +done + +disk_cache="$workspace"/.bazel_cache +mkdir -p "$disk_cache" + +# (>&2 echo :: exec "$bazel" "$@") +exec "$bazel" "$@" From a4a63c40f8e1134eeb0b828ee4bbe4d25e0f2f69 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:35:58 -0700 Subject: [PATCH 02/13] ensure one set of jobs is named test --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45fdd0682..c81f0cad2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,7 @@ dist: xenial sudo: required -language: generic -osx_image: xcode10.1 - -# Not technically required but suppresses 'Ruby' in Job status message. language: sh +osx_image: xcode10.1 cache: directories: @@ -33,7 +30,7 @@ _windows: &windows jobs: include: # test_rules_scala - - stage: test_rules_scala + - stage: test <<: *linux env: TEST_SCRIPT=test_rules_scala - <<: *osx From ab13aacb31d53e4aa9744f799317d01c70b6690e Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:38:50 -0700 Subject: [PATCH 03/13] Keep the original build structure --- .travis.yml | 59 ++++++++++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index c81f0cad2..eeea59079 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,59 +1,44 @@ -dist: xenial +# trusty beta image has jdk8, gcc4.8.4 +dist: trusty sudo: required +# xcode8 has jdk8 +osx_image: xcode8 +# Not technically required but suppresses 'Ruby' in Job status message. language: sh -osx_image: xcode10.1 cache: - directories: - - ~/.bazel_binaries - - .bazel_cache + directories: + - .bazel_cache + - ~/.bazel_binaries os: - linux - osx - windows -_linux: &linux - os: linux -_osx: &osx - os: osx - osx_image: xcode10.1 -_windows: &windows - os: windows - before_install: - - | - if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then - choco install jdk8 -params 'installdir=c:\\java8' - choco install bazel --version ${V} - fi +env: + # Linting is broken. Disable until fixed. + # See https://github.com/bazelbuild/rules_scala/pull/622 + # we want to test the last release + #- V=0.16.1 TEST_SCRIPT=test_lint.sh + - TEST_SCRIPT=test_rules_scala + #- V=0.14.1 TEST_SCRIPT=test_intellij_aspect.sh + - TEST_SCRIPT=test_reproducibility -jobs: - include: - # test_rules_scala - - stage: test - <<: *linux - env: TEST_SCRIPT=test_rules_scala - - <<: *osx - env: TEST_SCRIPT=test_rules_scala - - <<: *windows - env: TEST_SCRIPT=test_rules_scala - # test_reproducibility - - stage: test_reproducibility - <<: *linux - env: TEST_SCRIPT=test_reproducibility - - <<: *osx - env: TEST_SCRIPT=test_reproducibility - - <<: *windows - env: TEST_SCRIPT=test_reproducibility before_install: + - | + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + choco install jdk8 -params 'installdir=c:\\java8' + choco install bazel --version ${V} + fi - cat .bazelrc.travis >> .bazelrc script: - | if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then powershell -Command 'Set-ExecutionPolicy RemoteSigned -scope CurrentUser' - powershell -File ./${TEST_SCRIPT}.ps1 + BAZEL_VERSION=host powershell -File ./${TEST_SCRIPT}.ps1 else bash ./${TEST_SCRIPT}.sh ci fi From eb9d1bda6e5a2a88e6e54d6b39451863be257e6d Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:40:36 -0700 Subject: [PATCH 04/13] fix version for windows --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eeea59079..442d49037 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_install: - | if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then choco install jdk8 -params 'installdir=c:\\java8' - choco install bazel --version ${V} + choco install bazel --version 0.28.1 fi - cat .bazelrc.travis >> .bazelrc From 153571ede1adc59ed60d67c70a3fb70112e7c6bd Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:47:52 -0700 Subject: [PATCH 05/13] add ci stopgap to scripts used in ci --- test_reproducibility.sh | 3 +++ test_rules_scala.sh | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test_reproducibility.sh b/test_reproducibility.sh index 656857096..78965cefa 100755 --- a/test_reproducibility.sh +++ b/test_reproducibility.sh @@ -2,6 +2,9 @@ set -e +# ci stopgap: ensure tools/bazel is run for `bazel` invocations +PATH="$(cd "$(dirname "$0")"/..; pwd)"/tools:$PATH + md5_util() { if [[ "$OSTYPE" == "darwin"* ]]; then _md5_util="md5" diff --git a/test_rules_scala.sh b/test_rules_scala.sh index 9c2d41a59..ac0f84bf1 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -2,6 +2,9 @@ set -e +# ci stopgap: ensure tools/bazel is run for `bazel` invocations +PATH="$(cd "$(dirname "$0")"/..; pwd)"/tools:$PATH + test_disappearing_class() { git checkout test_expect_failure/disappearing_class/ClassProvider.scala bazel build test_expect_failure/disappearing_class:uses_class @@ -1138,4 +1141,4 @@ $runner test_scalac_jvm_flags_from_scala_toolchain_fails $runner test_scalac_jvm_flags_work_with_scalapb $runner test_scala_test_jvm_flags_on_target_overrides_toolchain_passes $runner test_scala_test_jvm_flags_from_scala_toolchain_passes -$runner test_scala_test_jvm_flags_from_scala_toolchain_fails \ No newline at end of file +$runner test_scala_test_jvm_flags_from_scala_toolchain_fails From ed70a26d5d920fd27531fdc259d8c6982eff2e3b Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 14:51:32 -0700 Subject: [PATCH 06/13] fix ci errors --- .travis.yml | 2 +- test_reproducibility.sh | 6 ++++-- test_rules_scala.sh | 6 ++++-- test_runner.sh | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 442d49037..f278cfacb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_install: - | if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then choco install jdk8 -params 'installdir=c:\\java8' - choco install bazel --version 0.28.1 + choco install bazel --version 0.28.0 fi - cat .bazelrc.travis >> .bazelrc diff --git a/test_reproducibility.sh b/test_reproducibility.sh index 78965cefa..917e5fbd7 100755 --- a/test_reproducibility.sh +++ b/test_reproducibility.sh @@ -2,8 +2,10 @@ set -e -# ci stopgap: ensure tools/bazel is run for `bazel` invocations -PATH="$(cd "$(dirname "$0")"/..; pwd)"/tools:$PATH +if ! bazel_loc="$(type -p 'bazel')" || [[ -z "$bazel_loc" ]]; then + export PATH="$(cd "$(dirname "$0")"; pwd)"/tools:$PATH + echo 'Using ./tools/bazel directly for bazel calls' +fi md5_util() { if [[ "$OSTYPE" == "darwin"* ]]; then diff --git a/test_rules_scala.sh b/test_rules_scala.sh index ac0f84bf1..ae5b30591 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -2,8 +2,10 @@ set -e -# ci stopgap: ensure tools/bazel is run for `bazel` invocations -PATH="$(cd "$(dirname "$0")"/..; pwd)"/tools:$PATH +if ! bazel_loc="$(type -p 'bazel')" || [[ -z "$bazel_loc" ]]; then + export PATH="$(cd "$(dirname "$0")"; pwd)"/tools:$PATH + echo 'Using ./tools/bazel directly for bazel calls' +fi test_disappearing_class() { git checkout test_expect_failure/disappearing_class/ClassProvider.scala diff --git a/test_runner.sh b/test_runner.sh index 33392676a..7e2825699 100644 --- a/test_runner.sh +++ b/test_runner.sh @@ -12,7 +12,7 @@ run_test_ci() { local TEST_ARG=$@ local log_file=output_$$.log echo "running test $TEST_ARG" - $TEST_ARG &>$log_file & + eval $TEST_ARG &>$log_file & local test_pid=$! SECONDS=0 test_pulse_printer $! $TIMOUT $TEST_ARG & @@ -83,4 +83,4 @@ get_test_runner() { exit 1 fi echo "run_test_${test_env}" -} \ No newline at end of file +} From 9c3051ba1940b061ace49f690ef73e079f873820 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 16:12:56 -0700 Subject: [PATCH 07/13] adjust for buildkite --- .bazelrc | 4 ---- tools/bazel | 31 ++++++------------------------- 2 files changed, 6 insertions(+), 29 deletions(-) delete mode 100644 .bazelrc diff --git a/.bazelrc b/.bazelrc deleted file mode 100644 index 1fa20d447..000000000 --- a/.bazelrc +++ /dev/null @@ -1,4 +0,0 @@ -common:v0.28 --config=noop - -# pick something trivial as a noop -common:noop --logging=3 \ No newline at end of file diff --git a/tools/bazel b/tools/bazel index 02198511d..89346faf5 100755 --- a/tools/bazel +++ b/tools/bazel @@ -5,7 +5,10 @@ workspace="$(cd "$(dirname "$0")"/..; pwd)" default_bazel_version='0.28.1' -case $(uname -s) in +if [ "$BUILDKITE" = true ]; then + bazel_version='host' +else + case $(uname -s) in Darwin|Linux) # shellcheck disable=SC2153 if [ -z "$BAZEL_VERSION" ]; then @@ -18,7 +21,8 @@ case $(uname -s) in # windows, presumably bazel_version='host' ;; -esac + esac +fi case "$bazel_version" in 'host') @@ -80,27 +84,4 @@ if [ -z "$bazel" ]; then fi fi -extra_command_args=() - -# shellcheck disable=SC2034 -IFS=. read -r major minor patch < <(echo "$bazel_version") -extra_command_args+=("--config=v$major.$minor") - -for (( i=1; i<=$#; i++ )) -do - case "${!i}" in - -*) - ;; - *) - n=$((i + 1)) - set -- "${@:1:$i}" "${extra_command_args[@]}" "${@:$n}" - break - ;; - esac -done - -disk_cache="$workspace"/.bazel_cache -mkdir -p "$disk_cache" - -# (>&2 echo :: exec "$bazel" "$@") exec "$bazel" "$@" From 01e6630389de2d62cf8bd54f5a3b652caa290e68 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 16:13:11 -0700 Subject: [PATCH 08/13] remove unused param --- tools/bazel | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/bazel b/tools/bazel index 89346faf5..ac9cf75b0 100755 --- a/tools/bazel +++ b/tools/bazel @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -workspace="$(cd "$(dirname "$0")"/..; pwd)" - default_bazel_version='0.28.1' if [ "$BUILDKITE" = true ]; then From 5b2ca135e98f4ef49c1fde9b8cba763f0c612eac Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 16:23:15 -0700 Subject: [PATCH 09/13] Tweak travis image config --- .travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index f278cfacb..c69d258cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,7 @@ -# trusty beta image has jdk8, gcc4.8.4 -dist: trusty +dist: xenial sudo: required -# xcode8 has jdk8 -osx_image: xcode8 -# Not technically required but suppresses 'Ruby' in Job status message. -language: sh +language: generic +osx_image: xcode10.1 cache: directories: From 9618c421533d3e6be99c4a5cfb6135a392e4b379 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 16:30:40 -0700 Subject: [PATCH 10/13] use sh language --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c69d258cb..a9f3e783c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ dist: xenial sudo: required -language: generic +language: sh osx_image: xcode10.1 cache: From e51381275e58109fb8994007d04adbb9c3dcb84c Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 17:39:30 -0700 Subject: [PATCH 11/13] libxml2-utils --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a9f3e783c..babe523a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ before_install: if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then choco install jdk8 -params 'installdir=c:\\java8' choco install bazel --version 0.28.0 + else + sudo apt-get install -y libxml2-utils fi - cat .bazelrc.travis >> .bazelrc From 2128059c77f8d630a40e4dfb4b71ca02b0ead20d Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 19:05:58 -0700 Subject: [PATCH 12/13] use apt addon --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index babe523a7..2b651f013 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,11 @@ sudo: required language: sh osx_image: xcode10.1 +addons: + apt: + packages: + - libxml2-utils + cache: directories: - .bazel_cache @@ -28,8 +33,6 @@ before_install: if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then choco install jdk8 -params 'installdir=c:\\java8' choco install bazel --version 0.28.0 - else - sudo apt-get install -y libxml2-utils fi - cat .bazelrc.travis >> .bazelrc From 4549f5b223854b86eda44a594eb94ea70e68cf05 Mon Sep 17 00:00:00 2001 From: Andy Scott Date: Wed, 14 Aug 2019 20:57:41 -0700 Subject: [PATCH 13/13] Add notes about updating bazel versions --- .travis.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b651f013..6a7389553 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,13 +18,34 @@ os: - osx - windows +### +# +# Want to update/change bazel versions? +# +# 1. Update the bazel_version case statement in +# ./tools/bazel to include hashes for the bazel version +# you're targeting. +# +# 2. either +# - If you're updating the default bazel version, change +# default_bazel_version in ./tools/bazel. +# or +# - If you want to add an additional bazel version to the build +# matrix, set BAZEL_VERSION= along side +# TEST_SCRIPT below. +# +# 3. If you need to update the Windows version, adjust +# the windows specific install code below. +# +### + env: # Linting is broken. Disable until fixed. # See https://github.com/bazelbuild/rules_scala/pull/622 # we want to test the last release - #- V=0.16.1 TEST_SCRIPT=test_lint.sh + #- TEST_SCRIPT=test_lint.sh - TEST_SCRIPT=test_rules_scala - #- V=0.14.1 TEST_SCRIPT=test_intellij_aspect.sh + #- TEST_SCRIPT=test_intellij_aspect.sh - TEST_SCRIPT=test_reproducibility