From 8e7b241a3b75414b06e2fbc85ae08ff3d8647281 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <Earle.Philhower.III@wdc.com>
Date: Fri, 21 Dec 2018 14:20:53 -0800
Subject: [PATCH 01/11] Cache core.a for CI builds

Hopefull give a significant speedup to the Arduino CI process.
---
 tests/common.sh | 6 +++++-
 tools/build.py  | 8 ++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index c62369fa38..976f7b82b8 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -42,7 +42,7 @@ function build_sketches()
     local build_mod=$4
     local build_rem=$5
     mkdir -p $build_dir
-    local build_cmd="python tools/build.py -b generic -v -w all -s 4M1M -v -k -p $PWD/$build_dir $build_arg "
+    local build_cmd="python tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir $build_arg "
     local sketches=$(find $srcpath -name *.ino | sort)
     print_size_info >size.log
     export ARDUINO_IDE_PATH=$arduino
@@ -219,6 +219,8 @@ if [ -z "$TRAVIS_BUILD_DIR" ]; then
     echo "TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR"
 fi
 
+cache_dir=$(mktemp -d)
+
 if [ "$BUILD_TYPE" = "build" ]; then
     install_arduino nodebug
     build_sketches_with_arduino 1 0
@@ -248,6 +250,8 @@ elif [ "$BUILD_TYPE" = "platformio_odd" ]; then
     build_sketches_with_platformio $TRAVIS_BUILD_DIR/libraries "--board nodemcuv2 --verbose" 2 1
 else
     echo "BUILD_TYPE not set or invalid"
+    rm -rf $cache_dir
     exit 1
 fi
 
+rm -rf $cache_dir
diff --git a/tools/build.py b/tools/build.py
index 88f8519714..27d95de724 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -29,11 +29,13 @@
 import tempfile
 import shutil
 
-def compile(tmp_dir, sketch, tools_dir, hardware_dir, ide_path, f, args):
+def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
     cmd = ide_path + '/arduino-builder '
     cmd += '-compile -logger=human '
     cmd += '-build-path "' + tmp_dir + '" '
     cmd += '-tools "' +  ide_path + '/tools-builder" '
+    if cache != "":
+        cmd += '-build-cache "' + cache + '" '
     if args.library_path:
         for lib_dir in args.library_path:
             cmd += '-libraries "' + lib_dir + '" '
@@ -95,6 +97,7 @@ def parse_args():
     parser.add_argument('--debug_port', help='Debug port',
                         choices=['Serial', 'Serial1'])
     parser.add_argument('--debug_level', help='Debug level')
+    parser.add_argument('--build_cache', help='Build directory to cache core.a', default='')
     parser.add_argument('sketch_path', help='Sketch file path')
     return parser.parse_args()
 
@@ -124,6 +127,7 @@ def main():
     if args.verbose:
         print("Sketch: ", sketch_path)
         print("Build dir: ", tmp_dir)
+        print("Cache dir: ", args.build_cache)
         print("Output: ", output_name)
 
     if args.verbose:
@@ -131,7 +135,7 @@ def main():
     else:
         f = open(tmp_dir + '/build.log', 'w')
 
-    res = compile(tmp_dir, sketch_path, tools_dir, hardware_dir, ide_path, f, args)
+    res = compile(tmp_dir, sketch_path, args.build_cache, tools_dir, hardware_dir, ide_path, f, args)
     if res != 0:
         return res
 

From 5dc16dbd02f439095342a6583d282487e84695a2 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Fri, 21 Dec 2018 15:24:52 -0800
Subject: [PATCH 02/11] Hack to ensure core.a looks newer during builds

Commented in the code, but buasically the generation of git version
includes on each build causes the cached core.a to be ignored
because that header is newer than the file we just wrote (even though
on any CI run the GIT version is identical).
---
 tests/common.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/common.sh b/tests/common.sh
index 976f7b82b8..00dab0c6eb 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -52,6 +52,14 @@ function build_sketches()
         if [ $testcnt -ne $build_rem ]; then
             continue  # Not ours to do
         fi
+
+        # Uber hack warning - setting the cached core.a file to the future to
+        # ensure it is accepted as unmodified during build process.  Problem is
+        # the git version header is rewritten after each compile, so the
+        # builder sees it is new and says "rebuild the whole thing!"
+
+        if [ -e $cache_dir/core/*.a ]; then touch -t 203712310000 $cache_dir/core/*.a; fi
+
         rm -rf $build_dir/*
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)

From 23085a3b6b109f95afdfb311e36ffc549466a9c5 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Fri, 21 Dec 2018 15:38:03 -0800
Subject: [PATCH 03/11] Try removing file updates

---
 platform.txt | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/platform.txt b/platform.txt
index 225c66fa57..6afd02b495 100644
--- a/platform.txt
+++ b/platform.txt
@@ -76,9 +76,12 @@ compiler.elf2hex.extra_flags=
 
 ## generate file with git version number
 ## needs bash, git, and echo
-recipe.hooks.core.prebuild.1.pattern=python "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
-recipe.hooks.core.prebuild.2.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_VER 0x`git --git-dir {runtime.platform.path}/.git rev-parse --short=8 HEAD 2>/dev/null || echo ffffffff` >{build.path}/core/core_version.h"
-recipe.hooks.core.prebuild.3.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_DESC `cd "{runtime.platform.path}"; git describe --tags 2>/dev/null || echo unix-{version}` >>{build.path}/core/core_version.h"
+recipe.hooks.core.prebuild.1.pattern=
+recipe.hooks.core.prebuild.2.pattern=
+recipe.hooks.core.prebuild.3.pattern=
+#recipe.hooks.core.prebuild.1.pattern=python "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
+#recipe.hooks.core.prebuild.2.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_VER 0x`git --git-dir {runtime.platform.path}/.git rev-parse --short=8 HEAD 2>/dev/null || echo ffffffff` >{build.path}/core/core_version.h"
+#recipe.hooks.core.prebuild.3.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_DESC `cd "{runtime.platform.path}"; git describe --tags 2>/dev/null || echo unix-{version}` >>{build.path}/core/core_version.h"
 
 ## windows-compatible version without git
 recipe.hooks.core.prebuild.1.pattern.windows=cmd.exe /c rem cannot sign on windows

From 2928c1bfecbeea565e5622dd65baf4527288c152 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Wed, 2 Jan 2019 18:14:29 -0800
Subject: [PATCH 04/11] Try and keep build.options.json

---
 tests/common.sh | 18 ++++++++++++++++--
 tools/build.py  |  1 +
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index 00dab0c6eb..02644a9209 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -58,9 +58,23 @@ function build_sketches()
         # the git version header is rewritten after each compile, so the
         # builder sees it is new and says "rebuild the whole thing!"
 
-        if [ -e $cache_dir/core/*.a ]; then touch -t 203712310000 $cache_dir/core/*.a; fi
+        if [ -e $cache_dir/core/*.a ]; then
+            echo "orig:"; cat $build_dir/build.options.json
+            head -8 $build_dir/build.options.json > $build_dir/../build.options.json
+            echo '  "sketchLocation" : "'$sketch'",' >> $build_dir/../build.options.json
+            tail -2 $build_dir/build.options.json >> $build_dir/../build.options.json
+            echo "patched:"; cat $build_dir/../build.options.json
+            mv $build_dir/core $build_dir/../savecore
+            touch -t 203712310000 $cache_dir/core/*.a
+        fi
+
+        rm -rf $build_dir/sketch
+
+        if [ -e $cache_dir/core/*.a ]; then
+            mv $build_dir/../build.options.json $build_dir/build.options.json
+            mv $build_dir/../savecore $build_dir/core
+        fi
 
-        rm -rf $build_dir/*
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
diff --git a/tools/build.py b/tools/build.py
index 27d95de724..eaae1f4b8d 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -57,6 +57,7 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
         cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
     cmd += ' '
     cmd += '-ide-version=10607 '
+    cmd += '-debug-level=10 '
     cmd += '-warnings={warnings} '.format(**vars(args))
     if args.verbose:
         cmd += '-verbose '

From ca545a25e9647c67162f06f08d2aea5e86149c8e Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Wed, 2 Jan 2019 18:14:29 -0800
Subject: [PATCH 05/11] Try and keep build.options.json

---
 tests/common.sh | 14 ++++++++++++--
 tools/build.py  |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index 00dab0c6eb..c6658b2dc4 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -58,9 +58,19 @@ function build_sketches()
         # the git version header is rewritten after each compile, so the
         # builder sees it is new and says "rebuild the whole thing!"
 
-        if [ -e $cache_dir/core/*.a ]; then touch -t 203712310000 $cache_dir/core/*.a; fi
+        if [ -e $cache_dir/core/*.a ]; then
+            head -8 $build_dir/build.options.json > $build_dir/../build.options.json
+            echo '  "sketchLocation" : "'$sketch'",' >> $build_dir/../build.options.json
+            tail -2 $build_dir/build.options.json >> $build_dir/../build.options.json
+        fi
+
+        rm -rf $build_dir/sketch
+
+        if [ -e $cache_dir/core/*.a ]; then
+            mv $build_dir/../build.options.json $build_dir/build.options.json
+            touch -t 203712310000 $cache_dir/core/*.a
+        fi
 
-        rm -rf $build_dir/*
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
diff --git a/tools/build.py b/tools/build.py
index 27d95de724..eaae1f4b8d 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -57,6 +57,7 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
         cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
     cmd += ' '
     cmd += '-ide-version=10607 '
+    cmd += '-debug-level=10 '
     cmd += '-warnings={warnings} '.format(**vars(args))
     if args.verbose:
         cmd += '-verbose '

From 8019bf4231c0ac5333a28a40e38ee2929771e679 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Wed, 2 Jan 2019 19:17:23 -0800
Subject: [PATCH 06/11] Undo platform.txt change

---
 platform.txt | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/platform.txt b/platform.txt
index 512e7b522c..d28b9c4b24 100644
--- a/platform.txt
+++ b/platform.txt
@@ -76,12 +76,9 @@ compiler.elf2hex.extra_flags=
 
 ## generate file with git version number
 ## needs bash, git, and echo
-recipe.hooks.core.prebuild.1.pattern=
-recipe.hooks.core.prebuild.2.pattern=
-recipe.hooks.core.prebuild.3.pattern=
-#recipe.hooks.core.prebuild.1.pattern=python "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
-#recipe.hooks.core.prebuild.2.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_VER 0x`git --git-dir {runtime.platform.path}/.git rev-parse --short=8 HEAD 2>/dev/null || echo ffffffff` >{build.path}/core/core_version.h"
-#recipe.hooks.core.prebuild.3.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_DESC `cd "{runtime.platform.path}"; git describe --tags 2>/dev/null || echo unix-{version}` >>{build.path}/core/core_version.h"
+recipe.hooks.core.prebuild.1.pattern=python "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
+recipe.hooks.core.prebuild.2.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_VER 0x`git --git-dir {runtime.platform.path}/.git rev-parse --short=8 HEAD 2>/dev/null || echo ffffffff` >{build.path}/core/core_version.h"
+recipe.hooks.core.prebuild.3.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_DESC `cd "{runtime.platform.path}"; git describe --tags 2>/dev/null || echo unix-{version}` >>{build.path}/core/core_version.h"
 
 ## windows-compatible version without git
 recipe.hooks.core.prebuild.1.pattern.windows=cmd.exe /c rem cannot sign on windows

From 59717850dc6de5c7905adb38b8eaed104abbf406 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Wed, 2 Jan 2019 19:19:14 -0800
Subject: [PATCH 07/11] Check output files

---
 tests/common.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/common.sh b/tests/common.sh
index aaf937a490..d35965eeda 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -101,6 +101,7 @@ function build_sketches()
                 cat build.log
             fi
         fi
+        cp build.log /tmp/build.log.1
         rm build.log
         print_size_info $build_dir/*.elf >>size.log
     done

From 5bfa5be067a123343bfb4efc1fa5dfc6bd817d71 Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Wed, 2 Jan 2019 19:24:20 -0800
Subject: [PATCH 08/11] Remove old elf/bin/maps from build dir

---
 tests/common.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index d35965eeda..23c8149ac3 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -65,7 +65,7 @@ function build_sketches()
             tail -2 $build_dir/build.options.json >> $build_dir/../build.options.json
         fi
 
-        rm -rf $build_dir/sketch
+        rm -rf $build_dir/sketch $build_dir/*.bin $build_dir/*.map $build_dir/*.elf
 
         if [ -e $cache_dir/core/*.a ]; then
             mv $build_dir/../build.options.json $build_dir/build.options.json
@@ -101,7 +101,6 @@ function build_sketches()
                 cat build.log
             fi
         fi
-        cp build.log /tmp/build.log.1
         rm build.log
         print_size_info $build_dir/*.elf >>size.log
     done

From a0cb206ed350bd4a10184f5bf54956572f2efa6b Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Thu, 3 Jan 2019 08:23:46 -0800
Subject: [PATCH 09/11] Cleanup comments explaining the build logic

---
 tests/common.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index 23c8149ac3..ea46114ceb 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -54,21 +54,26 @@ function build_sketches()
             continue  # Not ours to do
         fi
 
-        # Uber hack warning - setting the cached core.a file to the future to
-        # ensure it is accepted as unmodified during build process.  Problem is
-        # the git version header is rewritten after each compile, so the
-        # builder sees it is new and says "rebuild the whole thing!"
-
         if [ -e $cache_dir/core/*.a ]; then
+            # We need to preserve the build.options.json file and replace the last .ino
+            # with this sketch's ino file, or builder will throw everything away.  This
+            # head/tail easier than a complicated SED script to replace the 1 line we need.
             head -8 $build_dir/build.options.json > $build_dir/../build.options.json
             echo '  "sketchLocation" : "'$sketch'",' >> $build_dir/../build.options.json
             tail -2 $build_dir/build.options.json >> $build_dir/../build.options.json
         fi
 
+        # Clear out the last built sketch, map, elf, bin files, but leave the compiled
+        # objects in the core and libraries available for use so we don't need to rebuild
+        # them each sketch.
         rm -rf $build_dir/sketch $build_dir/*.bin $build_dir/*.map $build_dir/*.elf
 
         if [ -e $cache_dir/core/*.a ]; then
+            # Restore the new build.options.json file
             mv $build_dir/../build.options.json $build_dir/build.options.json
+            # Set the time of the cached core.a file to year 2037 so the GIT header
+            # we regen won't cause the builder to throw it out and rebuild from
+            # scratch.
             touch -t 203712310000 $cache_dir/core/*.a
         fi
 

From 038ede7ee7e2baf63acdd822b271fde13a38ae4f Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <Earle.Philhower.III@wdc.com>
Date: Thu, 3 Jan 2019 12:12:22 -0800
Subject: [PATCH 10/11] Clean up build.options.json and core.a adjustment

Use suggested SED line instead of head/tail to replace the .ino being built.

Use touch with a delta from now to ensure this script runs until the end
of time_t.
---
 tests/common.sh | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index ea46114ceb..78aae1e8d6 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -56,11 +56,11 @@ function build_sketches()
 
         if [ -e $cache_dir/core/*.a ]; then
             # We need to preserve the build.options.json file and replace the last .ino
-            # with this sketch's ino file, or builder will throw everything away.  This
-            # head/tail easier than a complicated SED script to replace the 1 line we need.
-            head -8 $build_dir/build.options.json > $build_dir/../build.options.json
-            echo '  "sketchLocation" : "'$sketch'",' >> $build_dir/../build.options.json
-            tail -2 $build_dir/build.options.json >> $build_dir/../build.options.json
+            # with this sketch's ino file, or builder will throw everything away.
+	    sed -i "s,^.*sketchLocation.*$, \"sketchLocation\": \"$sketch\"\,,g" $build_dir/build.options.json
+            # Set the time of the cached core.a file to the future so the GIT header
+            # we regen won't cause the builder to throw it out and rebuild from scratch.
+            touch -d 'now + 1 day' $cache_dir/core/*.a
         fi
 
         # Clear out the last built sketch, map, elf, bin files, but leave the compiled
@@ -68,15 +68,6 @@ function build_sketches()
         # them each sketch.
         rm -rf $build_dir/sketch $build_dir/*.bin $build_dir/*.map $build_dir/*.elf
 
-        if [ -e $cache_dir/core/*.a ]; then
-            # Restore the new build.options.json file
-            mv $build_dir/../build.options.json $build_dir/build.options.json
-            # Set the time of the cached core.a file to year 2037 so the GIT header
-            # we regen won't cause the builder to throw it out and rebuild from
-            # scratch.
-            touch -t 203712310000 $cache_dir/core/*.a
-        fi
-
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)

From f87edfe52fed956c45eaca0564b3577709f59a1a Mon Sep 17 00:00:00 2001
From: "Earle F. Philhower, III" <earlephilhower@yahoo.com>
Date: Sat, 5 Jan 2019 09:53:18 -0800
Subject: [PATCH 11/11] Remove unwanted debug change

---
 tools/build.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/build.py b/tools/build.py
index 963810d09d..828b53408e 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -58,7 +58,6 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
         cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
     cmd += ' '
     cmd += '-ide-version=10607 '
-    cmd += '-debug-level=10 '
     cmd += '-warnings={warnings} '.format(**vars(args))
     if args.verbose:
         cmd += '-verbose '