From c148de210fa4c640547774c3093944c054b03a49 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Mon, 12 Nov 2018 10:16:33 +0300
Subject: [PATCH 01/11] Make library for auto building and caching Homebrew
bottles
cache Homebrew
---
.travis.yml | 38 +++-
config.sh | 40 ++--
travis_osx_brew_cache.sh | 381 +++++++++++++++++++++++++++++++++++++++
3 files changed, 447 insertions(+), 12 deletions(-)
create mode 100644 travis_osx_brew_cache.sh
diff --git a/.travis.yml b/.travis.yml
index 93aea0c3..b281544f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,15 @@ dist: trusty
git:
submodules: false
+# https://docs.travis-ci.com/user/caching
+cache:
+ directories:
+ # https://stackoverflow.com/questions/39930171/cache-brew-builds-with-travis-ci
+ - $HOME/Library/Caches/Homebrew
+ - /usr/local/Homebrew/
+ # used in OSX custom build script dealing with local bottle caching
+ - $HOME/local_bottle_metadata
+
matrix:
fast_finish: true
include:
@@ -526,9 +535,36 @@ install: |
script: |
# Install and run tests
set -x
- install_run $PLAT
+ install_run $PLAT && rc=$? || rc=$?
set +x
+ #otherwise, Travis logic terminates prematurely
+ #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817
+ trap ERR
+
+ test "$rc" -eq 0
+
+before_cache: |
+ # Cleanup dirs to be cached
+ set -x
+ if [ -n "$IS_OSX" ]; then
+
+ # When Taps is cached, this dir causes "Error: file exists" on `brew update`
+ rm -rf "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask"
+ find "$(brew --repository)/Library/Taps" -type d -name .git -exec \
+ bash -xec '
+ cd $(dirname '\''{}'\'')
+ git status
+ # https://stackoverflow.com/questions/8296710/how-to-ignore-xargs-commands-if-stdin-input-is-empty/19038748#19038748
+ git ls-files --other -z | xargs -0 -n100 git add
+ git commit -a -m "Travis auto changes"
+ [[ -n $(git stash list) ]] && git stash drop' \;
+
+ brew_cache_cleanup
+
+ fi
+ set +x
+
after_success: |
# Upload wheels to pypi if requested
if [ -n "$TRAVIS_TAG" ]; then
diff --git a/config.sh b/config.sh
index 86d238e0..1534c1dc 100644
--- a/config.sh
+++ b/config.sh
@@ -24,28 +24,46 @@ else
echo " > Linux environment "
fi
+if [ -n "$IS_OSX" ]; then
+
+ source travis_osx_brew_cache.sh
+
+ BREW_SLOW_BUILIDING_PACKAGES=$(printf '%s\n' \
+ "x265 20" \
+ "cmake 15" \
+ "ffmpeg 10" \
+ )
+
+ #Contrib adds significantly to project's build time
+ if [ "$ENABLE_CONTRIB" -eq 1 ]; then
+ BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60))
+ fi
+
+fi
+
function pre_build {
echo "Starting pre-build"
- set -e
+ set -e -o pipefail
if [ -n "$IS_OSX" ]; then
echo "Running for OSX"
+
+ brew update --merge
+ brew_add_local_bottles
- brew update
+ # Don't query analytical info online on `brew info`,
+ # this takes several seconds and we don't need it
+ # see https://docs.brew.sh/Manpage , "info formula" section
+ export HOMEBREW_NO_GITHUB_API=1
echo 'Installing QT4'
brew tap | grep -qxF cartr/qt4 || brew tap -v cartr/qt4
brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin -v cartr/qt4
- brew list --versions qt@4 || brew install -v qt@4
- echo '-----------------'
- echo '-----------------'
+ brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; }
+
echo 'Installing FFmpeg'
- # brew install does produce output regularly on a regular MacOS,
- # but Travis doesn't see it for some reason
- brew list --versions ffmpeg || \
- travis_wait brew install -v ffmpeg --without-x264 --without-xvid --without-gpl
- brew info ffmpeg
- echo '-----------------'
+
+ brew_install_and_cache_within_time_limit ffmpeg || { [ $? -gt 1 ] && return 2 || return 0; }
else
echo "Running for linux"
diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh
new file mode 100644
index 00000000..c5f26190
--- /dev/null
+++ b/travis_osx_brew_cache.sh
@@ -0,0 +1,381 @@
+# Library to cache downloaded and locally-built Homebrew bottles in Travis OSX build.
+
+trap '{ sleep 1; #if we terminale too abruptly, Travis will lose some log output
+ exit 2; #The trap isn''t called in the parent function, so can''t use `return` here.
+ #`exit` will terminate the entire build but it seems we have no choice.
+}' ERR
+set -E
+
+#Should be in Travis' cache
+BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata"
+
+# Starting reference point for elapsed build time; seconds since the epoch.
+#TRAVIS_TIMER_START_TIME is set at the start of a log fold, in nanoseconds since the epoch
+BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9))
+
+# If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache
+# The cutoff moment should leave enough time till Travis' job time limit to process the main project.
+BREW_TIME_LIMIT=$((30*60))
+# If a slow-building package is about to be built and the projected build end moment is beyond this many seconds,
+# skip that build, fail the Travis job and save Travis cache.
+# This cutoff should leave enough time for before_cache and cache save.
+BREW_TIME_HARD_LIMIT=$((40*60))
+
+
+
+#Public functions
+
+function brew_install_and_cache_within_time_limit {
+ # Install the package and its dependencies one by one;
+ # use bottle if available, build and cache bottle if not.
+ # Terminate and exit with status 1 if this takes too long.
+ # Exit with status 2 on any other error.
+
+ local PACKAGE; PACKAGE="${1:?}"
+ local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT}
+ local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT}
+ local TIME_START;TIME_START=${4:-$BREW_TIME_START}
+
+ local BUILD_FROM_SOURCE INCLUDE_BUILD
+
+ _brew_is_bottle_available "$PACKAGE" || BUILD_FROM_SOURCE=1
+ [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true
+
+ # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay.
+ DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD`
+ for dep in $DEPS; do
+ #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation
+ #While we don't know that yet, we can make better use of Travis-given time with a laxer limit
+ #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache
+ brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $?
+ done
+
+ _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $?
+ _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)"
+ _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $?
+}
+
+function brew_add_local_bottles {
+ # Should be called after `brew update` at startup.
+ # Adds metadata for cached locally-built bottles to local formulas
+ # so that `brew` commands can find them.
+ # If the package was updated, removes the corresponding files
+ # and the bottle's entry in the formula, if any.
+
+ # Bottle entry in formula:
+ # bottle do
+ # <...>
+ # sha256 "<sha256>" => :<os_codename>
+ # <...>
+ # end
+
+ echo "Cached bottles:"
+ ls "$(brew --cache)/downloads" || true #may not exist initially since it's "$(brew --cache)" that is in Travis cache
+ echo "Saved .json's and links:"
+ ls "$BREW_LOCAL_BOTTLE_METADATA"
+
+ for JSON in "$BREW_LOCAL_BOTTLE_METADATA"/*.json; do
+ [ -e "$JSON" ] || break # OSX 10.11 bash has no nullglob
+ local PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH
+
+ _brew_parse_bottle_json "$JSON" PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH
+
+ echo "Adding local bottle: $PACKAGE ${JSON_VERSION}_${JSON_REBUILD}"
+
+ local FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH
+
+ _brew_parse_package_info "$PACKAGE" "$OS_CODENAME" FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH
+
+ local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true
+
+
+ local BOTTLE_LINK BOTTLE; BOTTLE_LINK="${JSON}.bottle.lnk";
+ local BOTTLE_EXISTS BOTTLE_MISMATCH VERSION_MISMATCH
+
+
+ # Check that the bottle file exists and is still appropriate for the formula
+ if [[ "$FORMULA_VERSION" != "$JSON_VERSION" || "$JSON_REBUILD" != "$FORMULA_REBUILD" ]]; then
+ VERSION_MISMATCH=1;
+ echo "The cached bottle is obsolete: formula ${FORMULA_VERSION}_${FORMULA_REBUILD}"
+ fi
+ if [ -f "$BOTTLE_LINK" ]; then
+ BOTTLE=$(cat "$BOTTLE_LINK");
+ BOTTLE=$(cd "$(dirname "$BOTTLE")"; pwd)/$(basename "$BOTTLE")
+
+ if [ -e "$BOTTLE" ]; then
+ BOTTLE_EXISTS=1;
+
+ # The hash in `brew --cache $PACKAGE` entry is generated from download URL,
+ # which itself is generated from base URL and version
+ # (see Homebrew/Library/Homebrew/download_strategy.rb:cached_location).
+ # So if version changes, hashes will always mismatch anyway
+ # and we don't need a separate message about this.
+ # XXX: OSX doesn't have `realpath` so can't compare the entire paths
+ if [ -n "$FORMULA_HAS_BOTTLE" -a -z "$VERSION_MISMATCH" -a \
+ "$(basename "$(brew --cache "$PACKAGE")")" != "$(basename "$BOTTLE")" ]; then
+ BOTTLE_MISMATCH=1;
+ echo "Cached bottle file doesn't correspond to formula's cache entry!" \
+ "This can happen if download URL has changed." >&2
+ fi
+ else
+ echo "Cached bottle file is missing!" >&2
+ fi
+ else
+ echo "Link file is missing or of invalid type!" >&2
+ fi
+
+ # Delete cached bottle and all metadata if invalid
+ if [[ -z "$BOTTLE_EXISTS" || -n "$VERSION_MISMATCH" || -n "$BOTTLE_MISMATCH" ]]; then
+ echo "Deleting the cached bottle and all metadata"
+
+ if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then
+ echo "A bottle block for the cached bottle was merged into the updated formula. Removing..."
+ local FORMULA; FORMULA=$(brew formula "$PACKAGE")
+ perl -wpe 'BEGIN { our $IN_BLOCK=0; }
+ if ( ($IN_BLOCK==0) && /^\s*bottle\s+do\s*$/ ) { $IN_BLOCK=1; next; }
+ if ( ($IN_BLOCK==1) && /^\s*end\s*$/ ) { $IN_BLOCK=-1; next; }
+ if ( ($IN_BLOCK==1) && /^\s*sha256\s+"(\w+)"\s+=>\s+:\w+\s*$/ )
+ { if ( $1 eq "'"$BOTTLE_HASH"'" ) {$_="";}; next; }
+ ' <"$FORMULA" >"${FORMULA}.new"
+ # Depending on diff version, 1 may mean differences found
+ # https://stackoverflow.com/questions/6971284/what-are-the-error-exit-values-for-diff
+ diff -u "$FORMULA" "${FORMULA}.new" || test $? -le 1
+ ( cd $(dirname "$FORMULA")
+ FORMULA=$(basename "$FORMULA")
+ mv -v "${FORMULA}.new" "$FORMULA"
+ git commit -m "Removed obsolete local bottle ${JSON_VERSION}_${JSON_REBUILD} :${OS_CODENAME}" "$FORMULA"
+ )
+ fi
+
+ if [ -n "$BOTTLE" ]; then rm "$BOTTLE"; fi
+ rm -f "$BOTTLE_LINK"
+ rm "$JSON"
+
+ #(Re)add metadata to the formula otherwise
+ else
+ if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then
+ echo "The cached bottle is already present in the formula"
+ else
+ brew bottle --merge --write "$JSON"
+ fi
+ fi
+ done
+}
+
+
+function brew_cache_cleanup {
+ #Cleanup caching directories
+ # Is supposed to be called in before_cache
+
+ #Lefovers from some failure probably
+ rm -f "$BREW_LOCAL_BOTTLE_METADATA"/*.tar.gz
+
+ #`brew cleanup` may delete locally-built bottles that weren't needed this time
+ # so we're saving and restoring them
+ local BOTTLE_LINK BOTTLE
+ for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+ BOTTLE=$(cat "$BOTTLE_LINK")
+ ln "$BOTTLE" "$BREW_LOCAL_BOTTLE_METADATA/"
+ done
+ brew cleanup
+ local BOTTLE_BASENAME
+ for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+ BOTTLE=$(cat "$BOTTLE_LINK")
+ BOTTLE_BASENAME=$(basename "$BOTTLE")
+ if test ! -e "$BOTTLE"; then
+ echo "Restoring: $BOTTLE_BASENAME"
+ mv "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME" "$BOTTLE"
+ else
+ rm "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME"
+ fi
+ done
+}
+
+
+function brew_go_bootstrap_mode {
+# Can be overridden
+# Terminate the build but ensure saving the cache
+
+ echo "Going into cache bootstrap mode"
+
+ #Can't just `exit` because that would terminate the build without saving the cache
+ #Have to replace further actions with no-ops
+
+ eval '
+ function '"$cmd"' { return 0; }
+ function repair_wheelhouse { return 0; }
+ function install_run {
+ echo -e "\nBuilding dependencies took too long. Restart the build in Travis UI to continue from cache.\n"
+
+ # Travis runs user scripts via `eval` i.e. in the same shell process.
+ # So have to unset errexit in order to get to cache save stage
+ set +e; return 1
+ }'
+}
+
+
+
+#Internal functions
+
+function _brew_parse_bottle_json {
+ # Parse JSON info about a package
+ # from `brew info --json=v1` input or a JSON file on stdin
+ # and save it into bash global variables specified in arguments
+
+ local JSON; JSON="${1:?}"; shift
+
+ local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True:
+ import sys,json; j=json.load(open(sys.argv[1],"rb")); [name]=j.keys(); [pdata]=j.values()
+ print name
+ print pdata["formula"]["pkg_version"]
+ print pdata["bottle"]["rebuild"]
+ [(tag_name, tag_dict)]=pdata["bottle"]["tags"].items()
+ print tag_name
+ print tag_dict["sha256"]
+ ' "$JSON")
+
+ unset JSON
+
+ { local i v; for i in {1..5}; do
+ read -r v
+ eval "${1:?}=\"$v\""
+ shift
+ done } <<< "$JSON_DATA"
+}
+
+function _brew_parse_package_info {
+ # Get and parse `brew info --json` about a package
+ # and save it into bash variables specified in arguments
+
+ local PACKAGE; PACKAGE="${1:?}"; shift
+ local OS_CODENAME;OS_CODENAME="${1:?}"; shift
+
+ local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True:
+ import sys, json, subprocess; j=json.loads(subprocess.check_output(("brew","info","--json=v1",sys.argv[1])))
+ data=j[0]
+ print data["versions"]["stable"]
+ bottle_data=data["bottle"]["stable"]
+ print bottle_data["rebuild"]
+ print bottle_data["files"].get(sys.argv[2],{"sha256":""})["sha256"]
+ ' \
+ "$PACKAGE" "$OS_CODENAME")
+
+ unset PACKAGE OS_CODENAME
+
+ { local i v; for i in {1..3}; do
+ read -r v
+ eval "${1:?}=\"$v\""
+ shift
+ done } <<< "$JSON_DATA"
+}
+
+
+
+function _brew_is_bottle_available {
+
+ local PACKAGE;PACKAGE="${1:?}"
+
+ local INFO="$(brew info "$PACKAGE" | head -n 1)"
+ if grep -qwF '(bottled)' <<<"$INFO"; then
+ echo "Bottle available: $INFO"
+ return 0
+ else
+ echo "Bottle not available: $INFO"
+ return 1
+ fi
+}
+
+function _brew_install_and_cache {
+ # Install bottle or make and cache bottle.
+ # assumes that deps were already installed.
+
+ local PACKAGE;PACKAGE="${1:?}"
+ local USE_BOTTLE;USE_BOTTLE="${2:?}"
+ local VERB
+
+ if brew list --versions "$PACKAGE"; then
+ if ! (brew outdated | grep -qx "$PACKAGE"); then
+ echo "Already the latest version: $PACKAGE"
+ return 0
+ fi
+ VERB=upgrade
+ else
+ VERB=install
+ fi
+
+ if [[ "$USE_BOTTLE" -gt 0 ]]; then
+ echo "Installing bottle for: $PACKAGE"
+ brew $VERB "$PACKAGE"
+ else
+ echo "Building bottle for: $PACKAGE"
+ brew $VERB --build-bottle "$PACKAGE"
+ exec 3>&1
+ local OUT=$(brew bottle --json "$PACKAGE" | tee /dev/fd/3)
+ exec 3>&-
+
+ ls "$PACKAGE"*
+ # doesn't seem to be a documented way to get file names
+ local BOTTLE; BOTTLE=$(grep -Ee '^./' <<<"$OUT")
+ #proper procedure as per https://discourse.brew.sh/t/how-are-bottle-and-postinstall-related-is-it-safe-to-run-bottle-after-postinstall/3410/4
+ brew uninstall "$PACKAGE"
+ brew install "$BOTTLE"
+
+ local JSON; JSON=$(sed -E 's/bottle(.[[:digit:]]+)?\.tar\.gz$/bottle.json/' <<<"$BOTTLE")
+
+ #`brew bottle --merge` doesn't return nonzero on nonexisting json file
+ test -f "$JSON" -a -f "$BOTTLE"
+
+ brew bottle --merge --write "$JSON"
+ local CACHED_BOTTLE; CACHED_BOTTLE="$(brew --cache "$PACKAGE")"
+ mv "$BOTTLE" "$CACHED_BOTTLE";
+ local CACHED_JSON; CACHED_JSON="${BREW_LOCAL_BOTTLE_METADATA}/$(basename "$JSON")"
+ mv "$JSON" "$CACHED_JSON"
+ #Symlinks aren't cached by Travis. Will just save paths in files then.
+ local BOTTLE_LINK; BOTTLE_LINK="${CACHED_JSON}.bottle.lnk"
+ echo "$CACHED_BOTTLE" >"$BOTTLE_LINK"
+
+ fi
+}
+
+
+
+
+function _brew_check_elapsed_build_time {
+ # If time limit has been reached,
+ # arrange for further build to be skipped and return 1
+
+ local TIME_START;TIME_START="${1:?}"
+ local TIME_LIMIT;TIME_LIMIT="${2:?}"
+
+ local ELAPSED_TIME;ELAPSED_TIME=$(($(date +%s) - $TIME_START))
+ echo "Elapsed time: "$(($ELAPSED_TIME/60))"m (${ELAPSED_TIME}s)"
+
+ if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then
+ brew_go_bootstrap_mode
+ return 1
+ fi
+ return 0
+}
+
+function _brew_check_slow_building_ahead {
+
+ #If the package's projected build completion is higher than hard limit,
+ # skip it and arrange for further build to be skipped and return 1
+
+ local PACKAGE="${1:?}"
+ local TIME_START="${2:?}"
+ local TIME_HARD_LIMIT="${3:?}"
+
+ PROJECTED_BUILD_TIME=$(echo "$BREW_SLOW_BUILIDING_PACKAGES" | awk '$1=="'"$PACKAGE"'"{print $2}')
+ [ -z "$PROJECTED_BUILD_TIME" ] && return 0 || true
+
+ local PROJECTED_BUILD_END_ELAPSED_TIME
+ PROJECTED_BUILD_END_ELAPSED_TIME=$(( $(date +%s) - TIME_START + PROJECTED_BUILD_TIME * 60))
+
+ if [[ "$PROJECTED_BUILD_END_ELAPSED_TIME" -ge "$TIME_HARD_LIMIT" ]]; then
+ echo -e "\nProjected build end elapsed time for $PACKAGE: $((PROJECTED_BUILD_END_ELAPSED_TIME/60))m ($PROJECTED_BUILD_END_ELAPSED_TIMEs)"
+ brew_go_bootstrap_mode
+ return 1
+ fi
+ return 0
+}
From 200f0da0e24e51c79176d655bf4da25a4a736119 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Mon, 12 Nov 2018 10:29:47 +0300
Subject: [PATCH 02/11] cache ccache
---
.travis.yml | 2 ++
config.sh | 6 ++++++
2 files changed, 8 insertions(+)
diff --git a/.travis.yml b/.travis.yml
index b281544f..fe9bf97f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,6 +33,8 @@ cache:
- /usr/local/Homebrew/
# used in OSX custom build script dealing with local bottle caching
- $HOME/local_bottle_metadata
+ # `cache: ccache: true` has no effect if `language:` is not `c` or `cpp`
+ - $HOME/.ccache
matrix:
fast_finish: true
diff --git a/config.sh b/config.sh
index 1534c1dc..88255c9d 100644
--- a/config.sh
+++ b/config.sh
@@ -56,6 +56,12 @@ function pre_build {
# see https://docs.brew.sh/Manpage , "info formula" section
export HOMEBREW_NO_GITHUB_API=1
+ # https://docs.travis-ci.com/user/caching/#ccache-cache
+ # No need to allow rc 1 -- if this triggers a timeout,
+ # something is clearly wrong
+ brew_install_and_cache_within_time_limit ccache
+ export PATH="/usr/local/opt/ccache/libexec:$PATH"
+
echo 'Installing QT4'
brew tap | grep -qxF cartr/qt4 || brew tap -v cartr/qt4
brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin -v cartr/qt4
From cf601a1a3866fac52bde2381a7448663acc5a128 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Wed, 14 Nov 2018 01:56:15 +0300
Subject: [PATCH 03/11] take advantage of multiple CPUs
---
config.sh | 2 ++
1 file changed, 2 insertions(+)
diff --git a/config.sh b/config.sh
index 88255c9d..559b0a65 100644
--- a/config.sh
+++ b/config.sh
@@ -20,8 +20,10 @@ function bdist_wheel_cmd {
if [ -n "$IS_OSX" ]; then
echo " > OSX environment "
+ export MAKEFLAGS="-j$(sysctl -n hw.ncpu)"
else
echo " > Linux environment "
+ export MAKEFLAGS="-j$(grep -E '^processor[[:space:]]*:' /proc/cpuinfo | wc -l)"
fi
if [ -n "$IS_OSX" ]; then
From 01df5ed7f717230f1ad07370531783190ef17700 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Fri, 16 Nov 2018 05:47:37 +0300
Subject: [PATCH 04/11] more accurate cleaning of Homebrew dirs
---
.travis.yml | 27 ++++++++++++++++++---------
travis_osx_brew_cache.sh | 2 ++
2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index fe9bf97f..8c54615e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -524,6 +524,21 @@ before_install: |
source multibuild_customize.sh
echo $ENABLE_CONTRIB > contrib.enabled
echo $ENABLE_HEADLESS > headless.enabled
+
+ if [ -n "$IS_OSX" ]; then
+ TAPS="$(brew --repository)/Library/Taps"
+ if [ -e "$TAPS/caskroom/homebrew-cask" -a -e "$TAPS/homebrew/homebrew-cask" ]; then
+ rm -rf "$TAPS/caskroom/homebrew-cask"
+ fi
+ find "$TAPS" -type d -name .git -exec \
+ bash -xec '
+ cd $(dirname '\''{}'\'')
+ git clean -fxd
+ git status' \;
+
+ brew_cache_cleanup
+ fi
+
before_install
# Not interested in travis internal scripts' output
set +x
@@ -552,15 +567,9 @@ before_cache: |
if [ -n "$IS_OSX" ]; then
# When Taps is cached, this dir causes "Error: file exists" on `brew update`
- rm -rf "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask"
- find "$(brew --repository)/Library/Taps" -type d -name .git -exec \
- bash -xec '
- cd $(dirname '\''{}'\'')
- git status
- # https://stackoverflow.com/questions/8296710/how-to-ignore-xargs-commands-if-stdin-input-is-empty/19038748#19038748
- git ls-files --other -z | xargs -0 -n100 git add
- git commit -a -m "Travis auto changes"
- [[ -n $(git stash list) ]] && git stash drop' \;
+ if [ -e "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask" ]; then
+ rm -rf "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask"
+ fi
brew_cache_cleanup
diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh
index c5f26190..2b4407ed 100644
--- a/travis_osx_brew_cache.sh
+++ b/travis_osx_brew_cache.sh
@@ -174,12 +174,14 @@ function brew_cache_cleanup {
# so we're saving and restoring them
local BOTTLE_LINK BOTTLE
for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+ [ -e "$BOTTLE_LINK" ] || break
BOTTLE=$(cat "$BOTTLE_LINK")
ln "$BOTTLE" "$BREW_LOCAL_BOTTLE_METADATA/"
done
brew cleanup
local BOTTLE_BASENAME
for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+ [ -e "$BOTTLE_LINK" ] || break
BOTTLE=$(cat "$BOTTLE_LINK")
BOTTLE_BASENAME=$(basename "$BOTTLE")
if test ! -e "$BOTTLE"; then
From c744b5af2e037f3c26a6a2c4d9fb9bc18cf1d4b9 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Tue, 20 Nov 2018 08:17:16 +0300
Subject: [PATCH 05/11] brew tap -v is no longer supported
---
config.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/config.sh b/config.sh
index 559b0a65..c9bf656a 100644
--- a/config.sh
+++ b/config.sh
@@ -65,8 +65,8 @@ function pre_build {
export PATH="/usr/local/opt/ccache/libexec:$PATH"
echo 'Installing QT4'
- brew tap | grep -qxF cartr/qt4 || brew tap -v cartr/qt4
- brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin -v cartr/qt4
+ brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4
+ brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4
brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; }
echo 'Installing FFmpeg'
From d364ab36e7190bec5712c9f9b4dc55ddcf460c34 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Tue, 20 Nov 2018 09:57:59 +0300
Subject: [PATCH 06/11] harden the source
---
travis_osx_brew_cache.sh | 38 ++++++++++++++++++++++----------------
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh
index 2b4407ed..47d37326 100644
--- a/travis_osx_brew_cache.sh
+++ b/travis_osx_brew_cache.sh
@@ -1,10 +1,12 @@
# Library to cache downloaded and locally-built Homebrew bottles in Travis OSX build.
-trap '{ sleep 1; #if we terminale too abruptly, Travis will lose some log output
+_BREW_ERREXIT='
+set -e -o pipefail
+trap '\''{ sleep 3; #if we terminale too abruptly, Travis will lose some log output
exit 2; #The trap isn''t called in the parent function, so can''t use `return` here.
#`exit` will terminate the entire build but it seems we have no choice.
-}' ERR
-set -E
+}'\'' ERR
+set -E'
#Should be in Travis' cache
BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata"
@@ -30,11 +32,12 @@ function brew_install_and_cache_within_time_limit {
# use bottle if available, build and cache bottle if not.
# Terminate and exit with status 1 if this takes too long.
# Exit with status 2 on any other error.
+ ( eval "$_BREW_ERREXIT"
- local PACKAGE; PACKAGE="${1:?}"
- local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT}
- local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT}
- local TIME_START;TIME_START=${4:-$BREW_TIME_START}
+ local PACKAGE; PACKAGE="${1:?}" || return 2
+ local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || return 2
+ local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || return 2
+ local TIME_START;TIME_START=${4:-$BREW_TIME_START} || return 2
local BUILD_FROM_SOURCE INCLUDE_BUILD
@@ -42,7 +45,7 @@ function brew_install_and_cache_within_time_limit {
[ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true
# Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay.
- DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD`
+ DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || return 2
for dep in $DEPS; do
#TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation
#While we don't know that yet, we can make better use of Travis-given time with a laxer limit
@@ -51,8 +54,10 @@ function brew_install_and_cache_within_time_limit {
done
_brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $?
- _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)"
+ _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)" || return 2
_brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $?
+ ) \
+ || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi #must run this in current process
}
function brew_add_local_bottles {
@@ -220,9 +225,8 @@ function brew_go_bootstrap_mode {
#Internal functions
function _brew_parse_bottle_json {
- # Parse JSON info about a package
- # from `brew info --json=v1` input or a JSON file on stdin
- # and save it into bash global variables specified in arguments
+ # Parse JSON file resulting from `brew bottle --json`
+ # and save data into specified variables
local JSON; JSON="${1:?}"; shift
@@ -247,7 +251,7 @@ function _brew_parse_bottle_json {
function _brew_parse_package_info {
# Get and parse `brew info --json` about a package
- # and save it into bash variables specified in arguments
+ # and save data into specified variables
local PACKAGE; PACKAGE="${1:?}"; shift
local OS_CODENAME;OS_CODENAME="${1:?}"; shift
@@ -255,12 +259,14 @@ function _brew_parse_package_info {
local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True:
import sys, json, subprocess; j=json.loads(subprocess.check_output(("brew","info","--json=v1",sys.argv[1])))
data=j[0]
- print data["versions"]["stable"]
+ revision=data["revision"]
+ # in bottle''s json, revision is included into version; here, they are separate
+ print data["versions"]["stable"]+("_"+str(revision) if revision else "")
bottle_data=data["bottle"]["stable"]
print bottle_data["rebuild"]
- print bottle_data["files"].get(sys.argv[2],{"sha256":""})["sha256"]
+ print bottle_data["files"].get(sys.argv[2],{"sha256":"!?"})["sha256"] #prevent losing trailing blank line to command substitution
' \
- "$PACKAGE" "$OS_CODENAME")
+ "$PACKAGE" "$OS_CODENAME"); JSON_DATA="${JSON_DATA%\!\?}" #!? can't occur in a hash
unset PACKAGE OS_CODENAME
From 4e59499843251e0ed1ef17acd81360502006647a Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Sat, 17 Nov 2018 01:31:28 +0300
Subject: [PATCH 07/11] use patched ffmpeg
---
config.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 45 insertions(+), 2 deletions(-)
diff --git a/config.sh b/config.sh
index c9bf656a..53c5ff2f 100644
--- a/config.sh
+++ b/config.sh
@@ -33,13 +33,55 @@ if [ -n "$IS_OSX" ]; then
BREW_SLOW_BUILIDING_PACKAGES=$(printf '%s\n' \
"x265 20" \
"cmake 15" \
- "ffmpeg 10" \
+ "ffmpeg_opencv 10" \
)
#Contrib adds significantly to project's build time
if [ "$ENABLE_CONTRIB" -eq 1 ]; then
BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60))
fi
+
+ function generate_ffmpeg_formula {
+ local FF="ffmpeg"
+ local LFF="ffmpeg_opencv"
+ local FF_FORMULA; FF_FORMULA=$(brew formula "$FF")
+ local LFF_FORMULA; LFF_FORMULA="$(dirname "$FF_FORMULA")/${LFF}.rb"
+
+ local REGENERATE
+ if [ -f "$LFF_FORMULA" ]; then
+ local UPSTREAM_VERSION VERSION
+ _brew_parse_package_info "$FF" " " UPSTREAM_VERSION _ _
+ _brew_parse_package_info "$LFF" " " VERSION _ _ || REGENERATE=1
+ #`rebuild` clause is ignored on `brew bottle` and deleted
+ # from newly-generated formula on `brew bottle --merge` for some reason
+ # so can't compare rebuild numbers
+ if [ "$UPSTREAM_VERSION" != "$VERSION" ]; then
+ REGENERATE=1
+ fi
+ else
+ REGENERATE=1
+ fi
+ if [ -n "$REGENERATE" ]; then
+ echo "Regenerating custom ffmpeg formula"
+ # Bottle block syntax: https://docs.brew.sh/Bottles#bottle-dsl-domain-specific-language
+ perl -wpe 'BEGIN {our ($found_blank, $bottle_block);}
+ if (/(^class )(Ffmpeg)(\s.*)/) {$_=$1.$2."Opencv".$3."\n"; next;}
+ if (!$found_blank && /^$/) {$_.="conflicts_with \"ffmpeg\"\n\n"; $found_blank=1; next;}
+ if (!$bottle_block && /^\s*bottle do$/) { $bottle_block=1; next; }
+ if ($bottle_block) { if (/^\s*end\s*$/) { $bottle_block=0} elsif (/^\s*sha256\s/) {$_=""} next; }
+if (/^\s*depends_on "(x264|x265|xvid)"$/) {$_=""; next;}
+ if (/^\s*--enable-(gpl|libx264|libx265|libxvid)$/) {$_=""; next;}
+ ' <"$FF_FORMULA" >"$LFF_FORMULA"
+ diff -u "$FF_FORMULA" "$LFF_FORMULA" || test $? -le 1
+
+ ( cd "$(dirname "$LFF_FORMULA")"
+ # This is the official way to add a formula
+ # https://docs.brew.sh/Formula-Cookbook#commit
+ git add "$(basename "$LFF_FORMULA")"
+ git commit -m "add/update custom ffmpeg ${VERSION}"
+ )
+ fi
+ }
fi
@@ -71,7 +113,8 @@ function pre_build {
echo 'Installing FFmpeg'
- brew_install_and_cache_within_time_limit ffmpeg || { [ $? -gt 1 ] && return 2 || return 0; }
+ generate_ffmpeg_formula
+ brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; }
else
echo "Running for linux"
From 313e09d2c9f5bd9bcbf7a5143e4246a646023694 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Tue, 20 Nov 2018 11:28:29 +0300
Subject: [PATCH 08/11] use ccache in Linux
fix: cache
---
.travis.yml | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 96 insertions(+)
diff --git a/.travis.yml b/.travis.yml
index 8c54615e..c231aab2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -218,18 +218,24 @@ matrix:
- MB_PYTHON_VERSION=2.7
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
@@ -237,34 +243,46 @@ matrix:
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
- TEST_DEPENDS=numpy==1.11.3
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
@@ -272,12 +290,16 @@ matrix:
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
- TEST_DEPENDS=numpy==1.11.3
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
- TEST_DEPENDS=numpy==1.14.5
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
@@ -285,6 +307,8 @@ matrix:
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=0
- TEST_DEPENDS=numpy==1.14.5
+ cache:
+ directories: $HOME/.ccache
# headless builds for Linux
- os: linux
@@ -292,18 +316,24 @@ matrix:
- MB_PYTHON_VERSION=2.7
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
@@ -311,34 +341,46 @@ matrix:
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- PLAT=i686
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
- TEST_DEPENDS=numpy==1.11.3
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
@@ -346,12 +388,16 @@ matrix:
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
- TEST_DEPENDS=numpy==1.11.3
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
- TEST_DEPENDS=numpy==1.14.5
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
@@ -359,6 +405,8 @@ matrix:
- ENABLE_CONTRIB=0
- ENABLE_HEADLESS=1
- TEST_DEPENDS=numpy==1.14.5
+ cache:
+ directories: $HOME/.ccache
# contrib builds for Linux
- os: linux
@@ -366,18 +414,24 @@ matrix:
- MB_PYTHON_VERSION=2.7
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
@@ -385,34 +439,46 @@ matrix:
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
- TEST_DEPENDS=numpy==1.11.3
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
@@ -420,12 +486,16 @@ matrix:
- TEST_DEPENDS=numpy==1.11.3
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
- TEST_DEPENDS=numpy==1.14.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
@@ -433,6 +503,8 @@ matrix:
- TEST_DEPENDS=numpy==1.14.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=0
+ cache:
+ directories: $HOME/.ccache
# headless contrib builds for Linux
@@ -441,18 +513,24 @@ matrix:
- MB_PYTHON_VERSION=2.7
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=2.7
@@ -460,34 +538,46 @@ matrix:
- UNICODE_WIDTH=16
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.4
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.5
- PLAT=i686
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
- TEST_DEPENDS=numpy==1.11.3
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.6
@@ -495,12 +585,16 @@ matrix:
- TEST_DEPENDS=numpy==1.11.3
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
- TEST_DEPENDS=numpy==1.14.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
- os: linux
env:
- MB_PYTHON_VERSION=3.7
@@ -508,6 +602,8 @@ matrix:
- TEST_DEPENDS=numpy==1.14.5
- ENABLE_CONTRIB=1
- ENABLE_HEADLESS=1
+ cache:
+ directories: $HOME/.ccache
# The first line is printed in the folding header in Travis output
before_install: |
From a27f9d565f56231663d95f64b55f2ad0519e51b1 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Tue, 20 Nov 2018 11:31:08 +0300
Subject: [PATCH 09/11] use multibuild facilities for ccache
---
.travis.yml | 2 +-
config.sh | 6 ------
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index c231aab2..d23b7c3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ env:
- TEST_DEPENDS="numpy==1.11.1"
# params to bdist_wheel. used to set osx build target.
- BDIST_PARAMS=""
-
+ - USE_CCACHE=1
- PLAT=x86_64
- UNICODE_WIDTH=32
- secure: "mU040XIYWtDjRms27deQy5fNg9HBFF9fiLfSteBaMjopZKXFUBsdMTKYGEVvX8DA879guMdjG8Prw1aCFhTTYlWgJ0Afm4YucRC0vAD4HLNhqLJ2lIpRceR3+2qPX+Oz5ATVVRi3ghBeJ20LLtNgOgf6esQVjdrYNC+YqmguClHKEYAxS7ngW42iQP8HX2anRcz9q9H7exZ9fX/D1PJfMNka/mNaB5KXZu5zdLuk/E0VbWU2tMWVIDUvx4uBlpE1d8HixEV5LHnuVE/QI36BcyucYxstTNKW6pGYgrhkYf+0PX4BphZXwY7EUBwzXsYLmyge6yH8W6NfvTW0ZasFF6xzQc9bsj+gAZN7H+hN2a42VQqIpkoJw9sU0hqzCOQf6ZvWUQgwFdAHJRHqe/zk4456WxnF0kAgbZdKaGOl0/n0WvgHNqD5bgO8Zzb1XyJTKoR+eAtYKXuz3KgpxKvZMMQVr8wMlI1cFEuGjIm+7ZrYB5jPvQrVzV/DgOq4gkPHOjjhu478UFlhGA9/XWwcyidC3b7zuBN2E7xVuTMlKdk7URB3AHXfG5bZgUG80vllQDGXQDpHVnv4Qi8bGCzI4iKTpp4fCibbqxFLxW1jhjmgePseGcie7Avpe+zXznkbmM2BqMCu3QRmtmFL3eCifwMf3rCNlAs0Sd3iLmEvyos="
diff --git a/config.sh b/config.sh
index 53c5ff2f..3bdacd2f 100644
--- a/config.sh
+++ b/config.sh
@@ -100,12 +100,6 @@ function pre_build {
# see https://docs.brew.sh/Manpage , "info formula" section
export HOMEBREW_NO_GITHUB_API=1
- # https://docs.travis-ci.com/user/caching/#ccache-cache
- # No need to allow rc 1 -- if this triggers a timeout,
- # something is clearly wrong
- brew_install_and_cache_within_time_limit ccache
- export PATH="/usr/local/opt/ccache/libexec:$PATH"
-
echo 'Installing QT4'
brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4
brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4
From 23f0ec06f5b728776ff42ad9311a6406c0e33803 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Tue, 20 Nov 2018 12:06:20 +0300
Subject: [PATCH 10/11] show ccache stats
---
.travis.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.travis.yml b/.travis.yml
index d23b7c3c..b4d114b1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -643,6 +643,7 @@ install: |
# Build and package
set -x
build_wheel $REPO_DIR $PLAT
+ if [ -n "$USE_CCACHE" ]; then ccache --show-stats; fi
set +x
script: |
From 8aaff83a74175efbd697c287035350295d6e5b92 Mon Sep 17 00:00:00 2001
From: Ivan Pozdeev <vano@mail.mipt.ru>
Date: Wed, 21 Nov 2018 03:38:22 +0300
Subject: [PATCH 11/11] fix "Not using downloaded repomd.xml because it is
older than what we have"
---
env_vars.sh | 5 +++++
1 file changed, 5 insertions(+)
create mode 100644 env_vars.sh
diff --git a/env_vars.sh b/env_vars.sh
new file mode 100644
index 00000000..d67c3523
--- /dev/null
+++ b/env_vars.sh
@@ -0,0 +1,5 @@
+if [ -n "$IS_OSX" ]; then
+ :
+else
+ yum clean metadata
+fi
\ No newline at end of file