From 97e56c426a8d8e2d26d9c826baaec4f6e4a8321a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 18 Apr 2017 15:54:39 +0200 Subject: [PATCH 01/68] First experiments with arduino-preprocessor --- builder.go | 4 +- hardware/platform.txt | 7 ++ preprocess_sketch.go | 109 ++++++++++++++++++++++++++++++++ test/helper_tools_downloader.go | 8 +++ test/tools_loader_test.go | 17 +++-- 5 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 preprocess_sketch.go diff --git a/builder.go b/builder.go index d5b8eff7..67d3e34f 100644 --- a/builder.go +++ b/builder.go @@ -89,7 +89,7 @@ func (s *Builder) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Generating function prototypes..."), - &ContainerAddPrototypes{}, + &PreprocessSketch{}, utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Compiling sketch..."), &RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX}, @@ -158,7 +158,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, - &ContainerAddPrototypes{}, + &PreprocessSketch{}, &PrintPreprocessedSource{}, } diff --git a/hardware/platform.txt b/hardware/platform.txt index ca8df1f7..767e7e7d 100644 --- a/hardware/platform.txt +++ b/hardware/platform.txt @@ -1,3 +1,10 @@ +# arduino-preprocessor +# -------------------- + +tools.arduino-preprocessor.path={runtime.tools.arduino-preprocessor.path} +tools.arduino-preprocessor.cmd.path={path}/bin/arduino-preprocessor +tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" -- -std=gnu++11 + # ctags # ------------------------------ tools.ctags.path={runtime.tools.ctags.path} diff --git a/preprocess_sketch.go b/preprocess_sketch.go new file mode 100644 index 00000000..6a819db3 --- /dev/null +++ b/preprocess_sketch.go @@ -0,0 +1,109 @@ +/* + * This file is part of Arduino Builder. + * + * Arduino Builder is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2015 Arduino LLC (http://www.arduino.cc/) + */ + +package builder + +import ( + "fmt" + "path/filepath" + + "io/ioutil" + + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" +) + +type PreprocessSketch struct{} + +func (s *PreprocessSketch) Run(ctx *types.Context) error { + sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + commands := []types.Command{ + &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, + &ArduinoPreprocessorRunner{}, + &SketchSaver{}, + } + + for _, command := range commands { + PrintRingNameIfDebug(ctx, command) + err := command.Run(ctx) + if err != nil { + return i18n.WrapError(err) + } + } + + return nil +} + +type ArduinoPreprocessorRunner struct{} + +func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { + buildProperties := ctx.BuildProperties + targetFilePath := ctx.FileToRead + logger := ctx.GetLogger() + + properties := buildProperties.Clone() + toolProps := buildProperties.SubTree("tools").SubTree("arduino-preprocessor") + properties.Merge(toolProps) + properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = targetFilePath + + pattern := properties[constants.BUILD_PROPERTIES_PATTERN] + if pattern == constants.EMPTY_STRING { + return i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, "arduino-preprocessor") + } + + commandLine := properties.ExpandPropsInString(pattern) + command, err := utils.PrepareCommand(commandLine, logger) + if err != nil { + return i18n.WrapError(err) + } + + verbose := ctx.Verbose + if verbose { + fmt.Println(commandLine) + } + + sourceBytes, err := command.Output() + if err != nil { + return i18n.WrapError(err) + } + + ctx.Source = string(sourceBytes) + if ctx.Source == "" { + // If the output is empty the original sketch may pass through + buf, err := ioutil.ReadFile(targetFilePath) + if err != nil { + return i18n.WrapError(err) + } + ctx.Source = string(buf) + } + //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", ctx.Source) + return nil +} diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index cfce881c..d9686bdd 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -113,6 +113,14 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }}, + Tool{Name: "arduino-preprocessor", Version: "0.0.1"}, //OsUrls: []OsUrl{ + // OsUrl{Os: "i686-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-pc-linux-gnu.tar.bz2"}, + // OsUrl{Os: "x86_64-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-pc-linux-gnu.tar.bz2"}, + // OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, + // OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, + // OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, + //} + } boardsManagerTools := []Tool{ diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index 3507afe8..c54f0426 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -30,11 +30,12 @@ package test import ( + "sort" + "testing" + "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/types" "github.com/stretchr/testify/require" - "sort" - "testing" ) type ByToolIDAndVersion []*types.Tool @@ -64,11 +65,15 @@ func TestLoadTools(t *testing.T) { NoError(t, err) tools := ctx.Tools - require.Equal(t, 6, len(tools)) + require.Equal(t, 7, len(tools)) sort.Sort(ByToolIDAndVersion(tools)) idx := 0 + require.Equal(t, "arduino-preprocessor", tools[idx].Name) + require.Equal(t, "0.0.1", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.0.1"), tools[idx].Folder) + idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder) @@ -136,7 +141,7 @@ func TestLoadLotsOfTools(t *testing.T) { NoError(t, err) tools := ctx.Tools - require.Equal(t, 8, len(tools)) + require.Equal(t, 9, len(tools)) sort.Sort(ByToolIDAndVersion(tools)) @@ -145,6 +150,10 @@ func TestLoadLotsOfTools(t *testing.T) { require.Equal(t, "4.0.0-atmel", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder) idx++ + require.Equal(t, "arduino-preprocessor", tools[idx].Name) + require.Equal(t, "0.0.1", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.0.1"), tools[idx].Folder) + idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder) From 7ca217b8556247f8687843706b1cb54b3be94dd7 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sat, 6 May 2017 17:38:22 +0200 Subject: [PATCH 02/68] Added code-completion support --- arduino-builder/main.go | 6 +++++- hardware/platform.txt | 2 +- preprocess_sketch.go | 39 +++++++++++++++++++++++++-------------- types/context.go | 2 ++ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index d2ca98c4..bf7bd6da 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -54,6 +54,7 @@ const VERSION = "1.3.25" const FLAG_ACTION_COMPILE = "compile" const FLAG_ACTION_PREPROCESS = "preprocess" const FLAG_ACTION_DUMP_PREFS = "dump-prefs" +const FLAG_ACTION_CODE_COMPLETE_AT = "code-complete-at" const FLAG_BUILD_OPTIONS_FILE = "build-options-file" const FLAG_HARDWARE = "hardware" const FLAG_TOOLS = "tools" @@ -118,6 +119,7 @@ func (h *propertiesFlag) Set(value string) error { var compileFlag *bool var preprocessFlag *bool var dumpPrefsFlag *bool +var codeCompleteAtFlag *string var buildOptionsFileFlag *string var hardwareFoldersFlag foldersFlag var toolsFoldersFlag foldersFlag @@ -141,6 +143,7 @@ func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") preprocessFlag = flag.Bool(FLAG_ACTION_PREPROCESS, false, "preprocess the given sketch") dumpPrefsFlag = flag.Bool(FLAG_ACTION_DUMP_PREFS, false, "dumps build properties used when compiling") + codeCompleteAtFlag = flag.String(FLAG_ACTION_CODE_COMPLETE_AT, "", "output code completions for sketch at a specific location. Location format is \"file:line:col\"") buildOptionsFileFlag = flag.String(FLAG_BUILD_OPTIONS_FILE, "", "Instead of specifying --"+FLAG_HARDWARE+", --"+FLAG_TOOLS+" etc every time, you can load all such options from a file") flag.Var(&hardwareFoldersFlag, FLAG_HARDWARE, "Specify a 'hardware' folder. Can be added multiple times for specifying multiple 'hardware' folders") flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders") @@ -330,7 +333,8 @@ func main() { if *dumpPrefsFlag { err = builder.RunParseHardwareAndDumpBuildProperties(ctx) - } else if *preprocessFlag { + } else if *preprocessFlag || *codeCompleteAtFlag != "" { + ctx.CodeCompleteAt = *codeCompleteAtFlag err = builder.RunPreprocess(ctx) } else { if flag.NArg() == 0 { diff --git a/hardware/platform.txt b/hardware/platform.txt index 767e7e7d..eabc6fa9 100644 --- a/hardware/platform.txt +++ b/hardware/platform.txt @@ -3,7 +3,7 @@ tools.arduino-preprocessor.path={runtime.tools.arduino-preprocessor.path} tools.arduino-preprocessor.cmd.path={path}/bin/arduino-preprocessor -tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" -- -std=gnu++11 +tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" "{codecomplete}" -- -std=gnu++11 # ctags # ------------------------------ diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 6a819db3..2fd7b291 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -33,8 +33,6 @@ import ( "fmt" "path/filepath" - "io/ioutil" - "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" @@ -48,7 +46,12 @@ func (s *PreprocessSketch) Run(ctx *types.Context) error { commands := []types.Command{ &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, &ArduinoPreprocessorRunner{}, - &SketchSaver{}, + } + + if ctx.CodeCompleteAt != "" { + commands = append(commands, &OutputCodeCompletions{}) + } else { + commands = append(commands, &SketchSaver{}) } for _, command := range commands { @@ -73,6 +76,11 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { toolProps := buildProperties.SubTree("tools").SubTree("arduino-preprocessor") properties.Merge(toolProps) properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = targetFilePath + if ctx.CodeCompleteAt != "" { + properties["codecomplete"] = "-output-code-completions=" + ctx.CodeCompleteAt + } else { + properties["codecomplete"] = "" + } pattern := properties[constants.BUILD_PROPERTIES_PATTERN] if pattern == constants.EMPTY_STRING { @@ -90,20 +98,23 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { fmt.Println(commandLine) } - sourceBytes, err := command.Output() + buf, err := command.Output() if err != nil { return i18n.WrapError(err) } - - ctx.Source = string(sourceBytes) - if ctx.Source == "" { - // If the output is empty the original sketch may pass through - buf, err := ioutil.ReadFile(targetFilePath) - if err != nil { - return i18n.WrapError(err) - } - ctx.Source = string(buf) + output := string(buf) + //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) + if ctx.CodeCompleteAt != "" { + ctx.CodeCompletions = output + } else { + ctx.Source = output } - //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", ctx.Source) + return nil +} + +type OutputCodeCompletions struct{} + +func (s *OutputCodeCompletions) Run(ctx *types.Context) error { + fmt.Println(ctx.CodeCompletions) return nil } diff --git a/types/context.go b/types/context.go index bc9491dd..1a69def3 100644 --- a/types/context.go +++ b/types/context.go @@ -18,6 +18,7 @@ type Context struct { SketchLocation string ArduinoAPIVersion string FQBN string + CodeCompleteAt string // Build options are serialized here BuildOptionsJson string @@ -53,6 +54,7 @@ type Context struct { Sketch *Sketch Source string SourceGccMinusE string + CodeCompletions string WarningsLevel string From f8e1038fa47a9b1db80abca21b29ff50de864d10 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 7 Jun 2017 17:10:43 +0200 Subject: [PATCH 03/68] Updated tools for arduino-preprocessor --- hardware/platform.txt | 2 +- test/helper_tools_downloader.go | 20 +++++++++++--------- test/tools_loader_test.go | 8 ++++---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/hardware/platform.txt b/hardware/platform.txt index eabc6fa9..144df2fb 100644 --- a/hardware/platform.txt +++ b/hardware/platform.txt @@ -2,7 +2,7 @@ # -------------------- tools.arduino-preprocessor.path={runtime.tools.arduino-preprocessor.path} -tools.arduino-preprocessor.cmd.path={path}/bin/arduino-preprocessor +tools.arduino-preprocessor.cmd.path={path}/arduino-preprocessor tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" "{codecomplete}" -- -std=gnu++11 # ctags diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index d9686bdd..a5f4b722 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -112,15 +112,17 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, - }}, - Tool{Name: "arduino-preprocessor", Version: "0.0.1"}, //OsUrls: []OsUrl{ - // OsUrl{Os: "i686-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-pc-linux-gnu.tar.bz2"}, - // OsUrl{Os: "x86_64-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-pc-linux-gnu.tar.bz2"}, - // OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, - // OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, - // OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, - //} - + }, + }, + Tool{Name: "arduino-preprocessor", Version: "0.1.0", + OsUrls: []OsUrl{ + // OsUrl{Os: "i686-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/arduino-preprocessor-0.1.0-x86_64-pc-linux-gnu.tar.bz2"}, + // OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, + // OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, + // OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, + }, + }, } boardsManagerTools := []Tool{ diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index c54f0426..a1ef40dc 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -71,8 +71,8 @@ func TestLoadTools(t *testing.T) { idx := 0 require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.0.1", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.0.1"), tools[idx].Folder) + require.Equal(t, "0.1.0", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.0"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) @@ -151,8 +151,8 @@ func TestLoadLotsOfTools(t *testing.T) { require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder) idx++ require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.0.1", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.0.1"), tools[idx].Folder) + require.Equal(t, "0.1.0", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.0"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) From d3759d78d8b200171f0bcc1dab3f2caa57c50887 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 8 May 2017 19:06:38 +0200 Subject: [PATCH 04/68] Export CMake project if the build is successfull The project contains: - all the needed files (core, precompiled and source libraries, preprocessed sketch) - a CMakeList.txt stub which should just work (not targeting any architecture except linux-x86_64 ATM) Next steps: produce a zip file and restrict the execution to suitable cores --- builder.go | 2 + constants/constants.go | 2 +- create_cmake_rule.go | 223 ++++++++++++++++++++++++++++++++++++ phases/libraries_builder.go | 2 +- utils/utils.go | 108 +++++++++++++++++ 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 create_cmake_rule.go diff --git a/builder.go b/builder.go index 67d3e34f..5955577d 100644 --- a/builder.go +++ b/builder.go @@ -128,6 +128,8 @@ func (s *Builder) Run(ctx *types.Context) error { &PrintUsedLibrariesIfVerbose{}, + &ExportProjectCMake{SketchError: mainErr != nil}, + &phases.Sizer{SketchError: mainErr != nil}, } otherErr := runCommands(ctx, commands, false) diff --git a/constants/constants.go b/constants/constants.go index d83ee273..e3612459 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -48,7 +48,7 @@ const BUILD_PROPERTIES_BUILD_SYSTEM_PATH = "build.system.path" const BUILD_PROPERTIES_BUILD_VARIANT = "build.variant" const BUILD_PROPERTIES_BUILD_VARIANT_PATH = "build.variant.path" const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" -const BUILD_PROPERTIES_COMPILER_C_ELF_EXTRAFLAGS = "compiler.c.elf.extra_flags" +const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" const BUILD_PROPERTIES_COMPILER_PATH = "compiler.path" const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" diff --git a/create_cmake_rule.go b/create_cmake_rule.go new file mode 100644 index 00000000..9d4e6cf8 --- /dev/null +++ b/create_cmake_rule.go @@ -0,0 +1,223 @@ +/* + * This file is part of Arduino Builder. + * + * Arduino Builder is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2015 Arduino LLC (http://www.arduino.cc/) + */ + +package builder + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/arduino/arduino-builder/builder_utils" + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" +) + +var VALID_EXPORT_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".s": true, ".a": true} +var DOTHEXTENSION = map[string]bool{".h": true, ".hh": true, ".hpp": true} +var DOTAEXTENSION = map[string]bool{".a": true} + +type ExportProjectCMake struct { + // Was there an error while compiling the sketch? + SketchError bool +} + +func (s *ExportProjectCMake) Run(ctx *types.Context) error { + //verbose := ctx.Verbose + logger := ctx.GetLogger() + + if s.SketchError { + return nil + } + + // Create new cmake subFolder - clean if the folder is already there + cmakeFolder := filepath.Join(ctx.BuildPath, "_cmake") + if _, err := os.Stat(cmakeFolder); err == nil { + os.RemoveAll(cmakeFolder) + } + os.Mkdir(cmakeFolder, 0777) + + // Create lib and build subfolders + libBaseFolder := filepath.Join(cmakeFolder, "lib") + os.Mkdir(libBaseFolder, 0777) + buildBaseFolder := filepath.Join(cmakeFolder, "build") + os.Mkdir(buildBaseFolder, 0777) + + // Create core subfolder path (don't create it yet) + coreFolder := filepath.Join(cmakeFolder, "core") + cmakeFile := filepath.Join(cmakeFolder, "CMakeLists.txt") + + // Copy used libraries in the correct folder + extensions := func(ext string) bool { return VALID_EXPORT_EXTENSIONS[ext] } + for _, library := range ctx.ImportedLibraries { + libFolder := filepath.Join(libBaseFolder, library.Name) + utils.CopyDir(library.Folder, libFolder, extensions) + // Remove examples folder + if _, err := os.Stat(filepath.Join(libFolder, "examples")); err == nil { + os.RemoveAll(filepath.Join(libFolder, "examples")) + } + } + + // Copy core + variant in use + preprocessed sketch in the correct folders + err := utils.CopyDir(ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH], coreFolder, extensions) + if err != nil { + fmt.Println(err) + } + err = utils.CopyDir(ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH], filepath.Join(coreFolder, "variant"), extensions) + if err != nil { + fmt.Println(err) + } + err = utils.CopyDir(ctx.SketchBuildPath, filepath.Join(cmakeFolder, "sketch"), extensions) + if err != nil { + fmt.Println(err) + } + + // Extract CFLAGS, CPPFLAGS and LDFLAGS + var defines []string + var linkerflags []string + var libs []string + var linkDirectories []string + + extractCompileFlags(ctx, constants.RECIPE_C_COMBINE_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + extractCompileFlags(ctx, constants.RECIPE_C_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + extractCompileFlags(ctx, constants.RECIPE_CPP_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + + // Extract folders with .h in them for adding in include list + var headerFiles []string + isHeader := func(ext string) bool { return DOTHEXTENSION[ext] } + utils.FindFilesInFolder(&headerFiles, cmakeFolder, isHeader, true) + foldersContainingDotH := findUniqueFoldersRelative(headerFiles, cmakeFolder) + + // Extract folders with .a in them for adding in static libs paths list + var staticLibsFiles []string + isStaticLib := func(ext string) bool { return DOTAEXTENSION[ext] } + utils.FindFilesInFolder(&staticLibsFiles, cmakeFolder, isStaticLib, true) + + // Generate the CMakeLists global file + + projectName := strings.TrimSuffix(filepath.Base(ctx.Sketch.MainFile.Name), filepath.Ext(ctx.Sketch.MainFile.Name)) + + cmakelist := "cmake_minimum_required(VERSION 3.5.0)\n" + cmakelist += "INCLUDE(FindPkgConfig)\n" + cmakelist += "project (" + projectName + " C CXX)\n" + cmakelist += "add_definitions (" + strings.Join(defines, " ") + " " + strings.Join(linkerflags, " ") + ")\n" + cmakelist += "include_directories (" + foldersContainingDotH + ")\n" + + // Make link directories relative + // We can totally discard them since they mostly are outside the core folder + // If they are inside the core they are not getting copied :) + var relLinkDirectories []string + for _, dir := range linkDirectories { + if strings.Contains(dir, cmakeFolder) { + relLinkDirectories = append(relLinkDirectories, strings.TrimPrefix(dir, cmakeFolder)) + } + } + + // Add SO_PATHS option for libraries not getting found by pkg_config + cmakelist += "set(EXTRA_LIBS_DIRS \"\" CACHE STRING \"Additional paths for dynamic libraries\")\n" + + for i, lib := range libs { + // Dynamic libraries should be discovered by pkg_config + lib = strings.TrimPrefix(lib, "-l") + libs[i] = lib + cmakelist += "pkg_search_module (" + strings.ToUpper(lib) + " " + lib + ")\n" + relLinkDirectories = append(relLinkDirectories, "${"+strings.ToUpper(lib)+"_LIBRARY_DIRS}") + } + cmakelist += "link_directories (" + strings.Join(relLinkDirectories, " ") + " ${EXTRA_LIBS_DIRS})\n" + for _, staticLibsFile := range staticLibsFiles { + // Static libraries are fully configured + lib := filepath.Base(staticLibsFile) + lib = strings.TrimPrefix(lib, "lib") + lib = strings.TrimSuffix(lib, ".a") + if !utils.SliceContains(libs, lib) { + libs = append(libs, lib) + cmakelist += "add_library (" + lib + " STATIC IMPORTED)\n" + location := strings.TrimPrefix(staticLibsFile, cmakeFolder) + cmakelist += "set_property(TARGET " + lib + " PROPERTY IMPORTED_LOCATION " + "${PROJECT_SOURCE_DIR}" + location + " )\n" + } + } + // Include source files + // TODO: remove .cpp and .h from libraries example folders + cmakelist += "file (GLOB_RECURSE SOURCES core/*.c* lib/*.c* sketch/*.c*)\n" + + // Compile and link project + cmakelist += "add_executable (" + projectName + " ${SOURCES} ${SOURCES_LIBS})\n" + cmakelist += "target_link_libraries( " + projectName + " " + strings.Join(libs, " ") + ")\n" + + utils.WriteFile(cmakeFile, cmakelist) + + return nil +} + +func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, linkerflags, linkDirectories *[]string, logger i18n.Logger) { + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, receipe, true, false, false, logger) + + for _, arg := range command.Args { + if strings.HasPrefix(arg, "-D") { + *defines = appendIfUnique(*defines, arg) + continue + } + if strings.HasPrefix(arg, "-l") { + *libs = appendIfUnique(*libs, arg) + continue + } + if strings.HasPrefix(arg, "-L") { + *linkDirectories = appendIfUnique(*linkDirectories, strings.TrimPrefix(arg, "-L")) + continue + } + if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") { + // HACK : from linkerflags remove MMD (no cache is produced) + if !strings.HasPrefix(arg, "-MMD") { + *linkerflags = appendIfUnique(*linkerflags, arg) + } + } + } +} + +func findUniqueFoldersRelative(slice []string, base string) string { + var out []string + for _, element := range slice { + path := filepath.Dir(element) + path = strings.TrimPrefix(path, base+"/") + if !utils.SliceContains(out, path) { + out = append(out, path) + } + } + return strings.Join(out, " ") +} + +func appendIfUnique(slice []string, element string) []string { + if !utils.SliceContains(slice, element) { + slice = append(slice, element) + } + return slice +} diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index fda56b80..d80fcf91 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -93,7 +93,7 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib name = strings.Replace(name, "lib", "", 1) libs_cmd += "-l" + name + " " } - ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_EXTRAFLAGS] += "\"-L" + path + "\" " + libs_cmd + ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LDFLAGS] += "\"-L" + path + "\" " + libs_cmd } } return nil diff --git a/utils/utils.go b/utils/utils.go index 318b3984..a5682c29 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -32,6 +32,8 @@ package utils import ( "crypto/md5" "encoding/hex" + "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -488,3 +490,109 @@ func ParseCppString(line string) (string, string, bool) { i += width } } + +// CopyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +// CopyDir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +// Symlinks are ignored and skipped. +func CopyDir(src string, dst string, extensions CheckExtensionFunc) (err error) { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf("destination already exists") + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = CopyDir(srcPath, dstPath, extensions) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + if extensions != nil && !extensions(strings.ToLower(filepath.Ext(srcPath))) { + continue + } + + err = CopyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return +} From 2fecce3971b864f2a7d1a442c85d3d7e27c4a4eb Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 14 Jun 2017 12:24:15 +0200 Subject: [PATCH 05/68] restrict cmake project generation to platforms declaring compiler.export_cmake --- constants/constants.go | 1 + create_cmake_rule.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/constants/constants.go b/constants/constants.go index e3612459..cca3ceaf 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -52,6 +52,7 @@ const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" const BUILD_PROPERTIES_COMPILER_PATH = "compiler.path" const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" +const BUILD_PROPERTIES_COMPILER_EXPORT_CMAKE_FLAGS = "compiler.export_cmake" const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst" const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local" const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc" diff --git a/create_cmake_rule.go b/create_cmake_rule.go index 9d4e6cf8..085efb91 100644 --- a/create_cmake_rule.go +++ b/create_cmake_rule.go @@ -55,7 +55,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { //verbose := ctx.Verbose logger := ctx.GetLogger() - if s.SketchError { + if s.SketchError || !canExportCmakeProject(ctx) { return nil } @@ -178,6 +178,10 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { return nil } +func canExportCmakeProject(ctx *types.Context) bool { + return ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_EXPORT_CMAKE_FLAGS] != "" +} + func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, linkerflags, linkDirectories *[]string, logger i18n.Logger) { command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, receipe, true, false, false, logger) From 471ff97357ff9d6297ecfba4e2a8f261078b75fb Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sun, 25 Jun 2017 23:45:11 +0200 Subject: [PATCH 06/68] update arduino-preprocessor to 0.1.1 --- test/helper_tools_downloader.go | 12 ++++++------ test/tools_loader_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index a5f4b722..dfbdb955 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -114,11 +114,11 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }, }, - Tool{Name: "arduino-preprocessor", Version: "0.1.0", + Tool{Name: "arduino-preprocessor", Version: "0.1.1", OsUrls: []OsUrl{ - // OsUrl{Os: "i686-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "x86_64-pc-linux-gnu", Url: "http://downloads.arduino.cc/tools/arduino-preprocessor-0.1.0-x86_64-pc-linux-gnu.tar.bz2"}, - // OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, + OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-cygwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"}, // OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, // OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }, @@ -708,9 +708,9 @@ func translateGOOSGOARCHToPackageIndexValue() []string { case "linux-386": return []string{"i686-pc-linux-gnu", "i686-linux-gnu"} case "windows-amd64": - return []string{"i686-mingw32"} + return []string{"i686-mingw32", "i686-cygwin"} case "windows-386": - return []string{"i686-mingw32"} + return []string{"i686-mingw32", "i686-cygwin"} case "darwin-amd64": return []string{"i386-apple-darwin11", "x86_64-apple-darwin"} case "linux-arm": diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index a1ef40dc..46036507 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -71,8 +71,8 @@ func TestLoadTools(t *testing.T) { idx := 0 require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.0", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.0"), tools[idx].Folder) + require.Equal(t, "0.1.1", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) @@ -151,8 +151,8 @@ func TestLoadLotsOfTools(t *testing.T) { require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder) idx++ require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.0", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.0"), tools[idx].Folder) + require.Equal(t, "0.1.1", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) From 007cfcf87850d8edbbbc719f06e105ece34af9d8 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 3 Aug 2017 11:10:39 +0200 Subject: [PATCH 07/68] add OSX and LinuxARM builds --- test/helper_tools_downloader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index dfbdb955..3b49513f 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -119,8 +119,8 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-linux-gnu.tar.bz2"}, OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-pc-linux-gnu.tar.bz2"}, OsUrl{Os: "i686-cygwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"}, - // OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, - // OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, + OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-apple-darwin11.tar.bz2"}, + OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-armhf-pc-linux-gnu.tar.bz2"}, }, }, } From 6c60c47c27cd20c02b4269a10b7f58581e5f0316 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 3 Aug 2017 11:45:52 +0200 Subject: [PATCH 08/68] change windows target (final trigger) --- test/helper_tools_downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index 3b49513f..3b422339 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -118,7 +118,7 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrls: []OsUrl{ OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-linux-gnu.tar.bz2"}, OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "i686-cygwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"}, OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-apple-darwin11.tar.bz2"}, OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-armhf-pc-linux-gnu.tar.bz2"}, }, From 3d06ecf8c0cd9de3ef61d2c6901eba1f09d9b20d Mon Sep 17 00:00:00 2001 From: copercini Date: Mon, 14 Aug 2017 16:17:43 -0300 Subject: [PATCH 09/68] Parallel compiling: arduino-builder will leverage your multi core pc If -jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes https://github.com/arduino/arduino-builder/issues/17 - Rebased from Federico Fissore branch https://github.com/arduino/arduino-builder/commit/0a3d888439ee369d7f4cc2333619e66fc7dcf672#diff-8f074e2b3fc911dbe8255891fc1a9b13R96 - Add HumanTagsLogger compatibility - Remove an if which suppress some GCC error messages here: https://github.com/arduino/arduino-builder/commit/0a3d888439ee369d7f4cc2333619e66fc7dcf672#diff-8f074e2b3fc911dbe8255891fc1a9b13R96 --- arduino-builder/main.go | 10 ++++++ builder_utils/utils.go | 61 ++++++++++++++++++++++++++++------ ctags_runner.go | 4 +-- i18n/i18n.go | 72 ++++++++++++++++++++++++++++++++++------- types/accessories.go | 34 +++++++++++++++++++ 5 files changed, 159 insertions(+), 22 deletions(-) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index bf7bd6da..8015fcd2 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -37,6 +37,7 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "syscall" @@ -80,6 +81,7 @@ const FLAG_LOGGER_HUMANTAGS = "humantags" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" const FLAG_VID_PID = "vid-pid" +const FLAG_JOBS = "jobs" type foldersFlag []string @@ -138,6 +140,7 @@ var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool var vidPidFlag *string +var jobsFlag *int func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -162,6 +165,7 @@ func init() { loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") + jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") } func main() { @@ -176,6 +180,12 @@ func main() { return } + if *jobsFlag > 0 { + runtime.GOMAXPROCS(*jobsFlag) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + ctx := &types.Context{} if *buildOptionsFileFlag != "" { diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 139ad770..b89346e4 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -31,15 +31,16 @@ package builder_utils import ( "bytes" - "fmt" "io" "os" "os/exec" "path/filepath" "strings" + "sync" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) @@ -146,15 +147,47 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { } func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { + if len(sources) == 0 { + return objectFiles, nil + } + objectFilesChan := make(chan string) + errorsChan := make(chan error) + doneChan := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(len(sources)) + for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) - if err != nil { + go func(source string) { + defer wg.Done() + objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + if err != nil { + errorsChan <- err + } else { + objectFilesChan <- objectFile + } + }(source) + } + + go func() { + wg.Wait() + doneChan <- struct{}{} + }() + + for { + select { + case objectFile := <-objectFilesChan: + objectFiles = append(objectFiles, objectFile) + case err := <-errorsChan: return nil, i18n.WrapError(err) + case <-doneChan: + close(objectFilesChan) + for objectFile := range objectFilesChan { + objectFiles = append(objectFiles, objectFile) + } + return objectFiles, nil } - - objectFiles = append(objectFiles, objectFile) } - return objectFiles, nil } func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { @@ -361,10 +394,20 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties } if echoOutput { - command.Stdout = os.Stdout + printToStdOut := func(data []byte) { + logger.UnformattedWrite(os.Stdout, data) + } + stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}} + defer stdout.Flush() + command.Stdout = stdout } - command.Stderr = os.Stderr + printToStdErr := func(data []byte) { + logger.UnformattedWrite(os.Stderr, data) + } + stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}} + defer stderr.Flush() + command.Stderr = stderr if echoOutput { err := command.Run() @@ -393,7 +436,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo } if echoCommandLine { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } return command, nil diff --git a/ctags_runner.go b/ctags_runner.go index 82e4165c..86bb3cfb 100644 --- a/ctags_runner.go +++ b/ctags_runner.go @@ -30,7 +30,7 @@ package builder import ( - "fmt" + "os" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/ctags" @@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { verbose := ctx.Verbose if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, err := command.Output() diff --git a/i18n/i18n.go b/i18n/i18n.go index ce85d22c..6532ccb1 100644 --- a/i18n/i18n.go +++ b/i18n/i18n.go @@ -38,12 +38,15 @@ import ( "regexp" "strconv" "strings" + "sync" ) var PLACEHOLDER = regexp.MustCompile("{(\\d)}") type Logger interface { Fprintln(w io.Writer, level string, format string, a ...interface{}) + UnformattedFprintln(w io.Writer, s string) + UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string } @@ -52,6 +55,10 @@ type NoopLogger struct{} func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {} +func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {} + +func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} + func (s NoopLogger) Println(level string, format string, a ...interface{}) {} func (s NoopLogger) Name() string { @@ -62,13 +69,21 @@ type HumanTagsLogger struct{} func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { format = "[" + level + "] " + format - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanTagsLogger) Name() string { return "humantags" } @@ -76,20 +91,48 @@ func (s HumanTagsLogger) Name() string { type HumanLogger struct{} func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanLogger) Name() string { return "human" } type MachineLogger struct{} -func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) { +func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + printMachineFormattedLogLine(w, level, format, a) +} + +func (s MachineLogger) Println(level string, format string, a ...interface{}) { + printMachineFormattedLogLine(os.Stdout, level, format, a) +} + +func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s MachineLogger) Name() string { + return "machine" +} + +func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) { a = append([]interface{}(nil), a...) for idx, value := range a { typeof := reflect.Indirect(reflect.ValueOf(value)).Kind() @@ -97,20 +140,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format a[idx] = url.QueryEscape(value.(string)) } } - fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a) - fmt.Fprintln(w) + fprintf(w, "===%s ||| %s ||| %s\n", level, format, a) } -func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - s.printWithoutFormatting(w, level, format, a) +var lock sync.Mutex + +func fprintln(w io.Writer, s string) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintln(w, s) } -func (s MachineLogger) Println(level string, format string, a ...interface{}) { - s.printWithoutFormatting(os.Stdout, level, format, a) +func write(w io.Writer, data []byte) { + lock.Lock() + defer lock.Unlock() + w.Write(data) } -func (s MachineLogger) Name() string { - return "machine" +func fprintf(w io.Writer, format string, a ...interface{}) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintf(w, format, a...) } func FromJavaToGoSyntax(s string) string { diff --git a/types/accessories.go b/types/accessories.go index 6601bbaa..4c87c30c 100644 --- a/types/accessories.go +++ b/types/accessories.go @@ -29,6 +29,11 @@ package types +import ( + "bytes" + "sync" +) + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -74,3 +79,32 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile { func (queue *UniqueSourceFileQueue) Empty() bool { return queue.Len() == 0 } + +type BufferedUntilNewLineWriter struct { + PrintFunc PrintFunc + Buffer bytes.Buffer + lock sync.Mutex +} + +type PrintFunc func([]byte) + +func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { + w.lock.Lock() + defer w.lock.Unlock() + + writtenToBuffer, err := w.Buffer.Write(p) + return writtenToBuffer, err +} + +func (w *BufferedUntilNewLineWriter) Flush() { + w.lock.Lock() + defer w.lock.Unlock() + + remainingBytes := w.Buffer.Bytes() + if len(remainingBytes) > 0 { + if remainingBytes[len(remainingBytes)-1] != '\n' { + remainingBytes = append(remainingBytes, '\n') + } + w.PrintFunc(remainingBytes) + } +} From 95215220ecb426cd37039e26403ade5fa0eb4059 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 15 Sep 2017 16:02:29 +0200 Subject: [PATCH 10/68] First experiments with RPC --- arduino-builder/main.go | 43 +++++++++++++ i18n/i18n.go | 47 ++++++++++++++ jsonrpc/rpc.go | 138 ++++++++++++++++++++++++++++++++++++++++ tools_loader.go | 15 ++++- types/context.go | 3 + 5 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 jsonrpc/rpc.go diff --git a/arduino-builder/main.go b/arduino-builder/main.go index 8015fcd2..8f2dfe88 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -41,9 +41,13 @@ import ( "strings" "syscall" + "runtime/pprof" + "runtime/trace" + "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/gohasissues" "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/jsonrpc" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" @@ -80,8 +84,10 @@ const FLAG_LOGGER_HUMAN = "human" const FLAG_LOGGER_HUMANTAGS = "humantags" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" +const FLAG_DAEMON = "daemon" const FLAG_VID_PID = "vid-pid" const FLAG_JOBS = "jobs" +const FLAG_TRACE = "trace" type foldersFlag []string @@ -139,8 +145,10 @@ var debugLevelFlag *int var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool +var daemonFlag *bool var vidPidFlag *string var jobsFlag *int +var traceFlag *bool func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -164,13 +172,39 @@ func init() { warningsLevelFlag = flag.String(FLAG_WARNINGS, "", "Sets warnings level. Available values are '"+FLAG_WARNINGS_NONE+"', '"+FLAG_WARNINGS_DEFAULT+"', '"+FLAG_WARNINGS_MORE+"' and '"+FLAG_WARNINGS_ALL+"'") loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") + daemonFlag = flag.Bool(FLAG_DAEMON, false, "daemonizes and serves its functions via rpc") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") + traceFlag = flag.Bool(FLAG_TRACE, false, "traces the whole process lifecycle") } func main() { + flag.Parse() + if *traceFlag { + f, err := os.Create("trace.out") + if err != nil { + panic(err) + } + defer f.Close() + + f2, err := os.Create("profile.out") + if err != nil { + panic(err) + } + defer f2.Close() + + pprof.StartCPUProfile(f2) + defer pprof.StopCPUProfile() + + err = trace.Start(f) + if err != nil { + panic(err) + } + defer trace.Stop() + } + if *versionFlag { fmt.Println("Arduino Builder " + VERSION) fmt.Println("Copyright (C) 2015 Arduino LLC and contributors") @@ -188,6 +222,15 @@ func main() { ctx := &types.Context{} + if *daemonFlag { + var loggerBuffer []string + logger := i18n.AccumulatorLogger{} + logger.Buffer = &loggerBuffer + //logger := i18n.HumanLogger{} + ctx.SetLogger(logger) + jsonrpc.RegisterAndServeJsonRPC(ctx) + } + if *buildOptionsFileFlag != "" { buildOptions := make(properties.Map) if _, err := os.Stat(*buildOptionsFileFlag); err == nil { diff --git a/i18n/i18n.go b/i18n/i18n.go index 6532ccb1..202d51c7 100644 --- a/i18n/i18n.go +++ b/i18n/i18n.go @@ -49,6 +49,7 @@ type Logger interface { UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string + Flush() string } type NoopLogger struct{} @@ -61,10 +62,44 @@ func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} func (s NoopLogger) Println(level string, format string, a ...interface{}) {} +func (s NoopLogger) Flush() string { + return "" +} + func (s NoopLogger) Name() string { return "noop" } +type AccumulatorLogger struct { + Buffer *[]string +} + +func (s AccumulatorLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + *s.Buffer = append(*s.Buffer, Format(format, a...)) +} + +func (s AccumulatorLogger) UnformattedFprintln(w io.Writer, str string) { + *s.Buffer = append(*s.Buffer, str) +} + +func (s AccumulatorLogger) UnformattedWrite(w io.Writer, data []byte) { + *s.Buffer = append(*s.Buffer, string(data)) +} + +func (s AccumulatorLogger) Println(level string, format string, a ...interface{}) { + s.Fprintln(nil, level, format, a...) +} + +func (s AccumulatorLogger) Flush() string { + str := strings.Join(*s.Buffer, "\n") + *s.Buffer = (*s.Buffer)[0:0] + return str +} + +func (s AccumulatorLogger) Name() string { + return "accumulator" +} + type HumanTagsLogger struct{} func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { @@ -84,6 +119,10 @@ func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) { write(w, data) } +func (s HumanTagsLogger) Flush() string { + return "" +} + func (s HumanTagsLogger) Name() string { return "humantags" } @@ -106,6 +145,10 @@ func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { write(w, data) } +func (s HumanLogger) Flush() string { + return "" +} + func (s HumanLogger) Name() string { return "human" } @@ -124,6 +167,10 @@ func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { fprintln(w, str) } +func (s MachineLogger) Flush() string { + return "" +} + func (s MachineLogger) Name() string { return "machine" } diff --git a/jsonrpc/rpc.go b/jsonrpc/rpc.go new file mode 100644 index 00000000..7acfa162 --- /dev/null +++ b/jsonrpc/rpc.go @@ -0,0 +1,138 @@ +package jsonrpc + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + + builder "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/types" + "github.com/fsnotify/fsnotify" + "github.com/osamingo/jsonrpc" + "golang.org/x/net/context" +) + +type ( + BuildHandler struct { + watcher *fsnotify.Watcher + ctx *types.Context + } + BuildParams struct { + HardwareFolders string + ToolsFolders string + BuiltInLibrariesFolders string + OtherLibrariesFolders string + SketchLocation string + FQBN string + ArduinoAPIVersion string + CustomBuildProperties string + Verbose bool + BuildCachePath string + BuildPath string + WarningsLevel string + } + BuildResult struct { + Message string `json:"message"` + Error error + } +) + +type ( + WatchHandler struct { + watcher *fsnotify.Watcher + } + WatchParams struct { + Path string `json:"path"` + } + WatchResult struct { + Message string `json:"message"` + } +) + +func (h *BuildHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { + + var p BuildParams + if err := jsonrpc.Unmarshal(params, &p); err != nil { + fmt.Println(err) + return nil, err + } + + h.ctx.HardwareFolders = strings.Split(p.HardwareFolders, ",") + h.ctx.ToolsFolders = strings.Split(p.ToolsFolders, ",") + h.ctx.BuiltInLibrariesFolders = strings.Split(p.BuiltInLibrariesFolders, ",") + h.ctx.OtherLibrariesFolders = strings.Split(p.OtherLibrariesFolders, ",") + h.ctx.SketchLocation = p.SketchLocation + h.ctx.CustomBuildProperties = strings.Split(p.CustomBuildProperties, ",") + h.ctx.ArduinoAPIVersion = p.ArduinoAPIVersion + h.ctx.FQBN = p.FQBN + h.ctx.Verbose = p.Verbose + h.ctx.BuildCachePath = p.BuildCachePath + h.ctx.BuildPath = p.BuildPath + h.ctx.WarningsLevel = p.WarningsLevel + + err := builder.RunBuilder(h.ctx) + if err != nil { + return BuildResult{ + Message: h.ctx.GetLogger().Flush(), + Error: err, + }, nil + } + + return BuildResult{ + Message: h.ctx.GetLogger().Flush(), + }, nil +} + +func (h *WatchHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { + + var p WatchParams + if err := jsonrpc.Unmarshal(params, &p); err != nil { + return nil, err + } + + err := h.watcher.Add(p.Path) + if err != nil { + return nil, jsonrpc.ErrInvalidParams() + } + return BuildResult{ + Message: "OK " + p.Path, + }, nil +} + +func startWatching(ctx *types.Context) *fsnotify.Watcher { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + go func() { + for { + select { + case event := <-watcher.Events: + ctx.CanUseCachedTools = false + log.Println("event:", event) + case err := <-watcher.Errors: + log.Println("error:", err) + } + } + }() + return watcher +} + +func RegisterAndServeJsonRPC(ctx *types.Context) { + + watcher := startWatching(ctx) + + jsonrpc.RegisterMethod("Main.Build", &BuildHandler{watcher, ctx}, BuildParams{}, BuildResult{}) + jsonrpc.RegisterMethod("Main.AddWatchPath", &WatchHandler{watcher}, WatchParams{}, WatchResult{}) + + http.HandleFunc("/jrpc", func(w http.ResponseWriter, r *http.Request) { + jsonrpc.HandlerFunc(r.Context(), w, r) + }) + http.HandleFunc("/jrpc/debug", jsonrpc.DebugHandlerFunc) + if err := http.ListenAndServe(":8888", nil); err != nil { + log.Fatalln(err) + } +} diff --git a/tools_loader.go b/tools_loader.go index 2d9ee00a..d4f9c233 100644 --- a/tools_loader.go +++ b/tools_loader.go @@ -30,14 +30,16 @@ package builder import ( + "fmt" + "os" + "path/filepath" + "strings" + "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/gohasissues" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "os" - "path/filepath" - "strings" ) type ToolsLoader struct{} @@ -47,6 +49,11 @@ func (s *ToolsLoader) Run(ctx *types.Context) error { tools := []*types.Tool{} + if ctx.CanUseCachedTools { + fmt.Println("no fs modification, can use cached ctx") + return nil + } + for _, folder := range folders { builtinToolsVersionsFile, err := findBuiltinToolsVersionsFile(folder) if err != nil { @@ -82,6 +89,8 @@ func (s *ToolsLoader) Run(ctx *types.Context) error { } } + ctx.CanUseCachedTools = true + ctx.Tools = tools return nil diff --git a/types/context.go b/types/context.go index 1a69def3..5801844f 100644 --- a/types/context.go +++ b/types/context.go @@ -90,6 +90,9 @@ type Context struct { // ReadFileAndStoreInContext command FileToRead string + + // Reuse old tools since the backing storage didn't change + CanUseCachedTools bool } func (ctx *Context) ExtractBuildOptions() properties.Map { From f9e1acbce35640fdf02bc72e1b44d492c8def731 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 15 Sep 2017 17:02:35 +0200 Subject: [PATCH 11/68] cleanup needed ctx sections between runs --- jsonrpc/rpc.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jsonrpc/rpc.go b/jsonrpc/rpc.go index 7acfa162..db6cedb2 100644 --- a/jsonrpc/rpc.go +++ b/jsonrpc/rpc.go @@ -71,6 +71,14 @@ func (h *BuildHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) h.ctx.BuildCachePath = p.BuildCachePath h.ctx.BuildPath = p.BuildPath h.ctx.WarningsLevel = p.WarningsLevel + h.ctx.PrototypesSection = "" + + h.ctx.IncludeFolders = h.ctx.IncludeFolders[0:0] + h.ctx.LibrariesObjectFiles = h.ctx.LibrariesObjectFiles[0:0] + h.ctx.CoreObjectsFiles = h.ctx.CoreObjectsFiles[0:0] + h.ctx.SketchObjectFiles = h.ctx.SketchObjectFiles[0:0] + + h.ctx.ImportedLibraries = h.ctx.ImportedLibraries[0:0] err := builder.RunBuilder(h.ctx) if err != nil { From 7c70931b153f86c99389ff94e99e6b1683a8fbed Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 15 Sep 2017 17:18:18 +0200 Subject: [PATCH 12/68] Implement autocomplete endpoint --- jsonrpc/rpc.go | 70 ++++++++++++++++++++++++++++++++++++++++++++ preprocess_sketch.go | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/jsonrpc/rpc.go b/jsonrpc/rpc.go index db6cedb2..3992f388 100644 --- a/jsonrpc/rpc.go +++ b/jsonrpc/rpc.go @@ -39,6 +39,32 @@ type ( } ) +type ( + CompleteHandler struct { + watcher *fsnotify.Watcher + ctx *types.Context + } + CompleteParams struct { + HardwareFolders string + ToolsFolders string + BuiltInLibrariesFolders string + OtherLibrariesFolders string + SketchLocation string + FQBN string + ArduinoAPIVersion string + CustomBuildProperties string + Verbose bool + BuildCachePath string + BuildPath string + WarningsLevel string + CodeCompleteAt string + } + CompleteResult struct { + Message string `json:"message"` + Error error + } +) + type ( WatchHandler struct { watcher *fsnotify.Watcher @@ -51,6 +77,49 @@ type ( } ) +func (h *CompleteHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { + + var p CompleteParams + if err := jsonrpc.Unmarshal(params, &p); err != nil { + fmt.Println(err) + return nil, err + } + + h.ctx.HardwareFolders = strings.Split(p.HardwareFolders, ",") + h.ctx.ToolsFolders = strings.Split(p.ToolsFolders, ",") + h.ctx.BuiltInLibrariesFolders = strings.Split(p.BuiltInLibrariesFolders, ",") + h.ctx.OtherLibrariesFolders = strings.Split(p.OtherLibrariesFolders, ",") + h.ctx.SketchLocation = p.SketchLocation + h.ctx.CustomBuildProperties = strings.Split(p.CustomBuildProperties, ",") + h.ctx.ArduinoAPIVersion = p.ArduinoAPIVersion + h.ctx.FQBN = p.FQBN + h.ctx.Verbose = false //p.Verbose + h.ctx.BuildCachePath = p.BuildCachePath + h.ctx.BuildPath = p.BuildPath + h.ctx.WarningsLevel = p.WarningsLevel + h.ctx.PrototypesSection = "" + h.ctx.CodeCompleteAt = p.CodeCompleteAt + + h.ctx.IncludeFolders = h.ctx.IncludeFolders[0:0] + h.ctx.LibrariesObjectFiles = h.ctx.LibrariesObjectFiles[0:0] + h.ctx.CoreObjectsFiles = h.ctx.CoreObjectsFiles[0:0] + h.ctx.SketchObjectFiles = h.ctx.SketchObjectFiles[0:0] + + h.ctx.ImportedLibraries = h.ctx.ImportedLibraries[0:0] + + err := builder.RunPreprocess(h.ctx) + if err != nil { + return BuildResult{ + Message: h.ctx.GetLogger().Flush(), + Error: err, + }, nil + } + + return CompleteResult{ + Message: h.ctx.GetLogger().Flush(), + }, nil +} + func (h *BuildHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { var p BuildParams @@ -134,6 +203,7 @@ func RegisterAndServeJsonRPC(ctx *types.Context) { watcher := startWatching(ctx) jsonrpc.RegisterMethod("Main.Build", &BuildHandler{watcher, ctx}, BuildParams{}, BuildResult{}) + jsonrpc.RegisterMethod("Main.Complete", &CompleteHandler{watcher, ctx}, CompleteParams{}, CompleteResult{}) jsonrpc.RegisterMethod("Main.AddWatchPath", &WatchHandler{watcher}, WatchParams{}, WatchResult{}) http.HandleFunc("/jrpc", func(w http.ResponseWriter, r *http.Request) { diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 2fd7b291..ee459939 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -115,6 +115,6 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { type OutputCodeCompletions struct{} func (s *OutputCodeCompletions) Run(ctx *types.Context) error { - fmt.Println(ctx.CodeCompletions) + ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, "%s", ctx.CodeCompletions) return nil } From cab702e7fa652dc4b3672d0ada6cff1e53cd1d00 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 18 Sep 2017 13:21:14 +0200 Subject: [PATCH 13/68] Initial implementation using grps with streaming capabilities --- arduino-builder/main.go | 2 +- client/client.go | 87 ++++++++++ grpc/proto/builder.pb.go | 343 +++++++++++++++++++++++++++++++++++++++ grpc/proto/builder.proto | 57 +++++++ grpc/rpc.go | 180 ++++++++++++++++++++ jsonrpc/rpc.go | 216 ------------------------ 6 files changed, 668 insertions(+), 217 deletions(-) create mode 100644 client/client.go create mode 100644 grpc/proto/builder.pb.go create mode 100644 grpc/proto/builder.proto create mode 100644 grpc/rpc.go delete mode 100644 jsonrpc/rpc.go diff --git a/arduino-builder/main.go b/arduino-builder/main.go index 8f2dfe88..ae4a92d1 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -46,8 +46,8 @@ import ( "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/gohasissues" + "github.com/arduino/arduino-builder/grpc" "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/jsonrpc" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" diff --git a/client/client.go b/client/client.go new file mode 100644 index 00000000..9576073d --- /dev/null +++ b/client/client.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a simple gRPC client that demonstrates how to use gRPC-Go libraries +// to perform unary, client streaming, server streaming and full duplex RPCs. +// +// It interacts with the route guide service whose definition can be found in routeguide/route_guide.proto. +package main + +import ( + "io" + "log" + + pb "arduino.cc/builder/grpc/proto" + "golang.org/x/net/context" + "google.golang.org/grpc" +) + +// printFeature gets the feature for the given point. +func autocomplete(client pb.BuilderClient, in *pb.BuildParams) { + resp, err := client.Autocomplete(context.Background(), in) + if err != nil { + log.Fatalf("%v.GetFeatures(_) = _, %v: ", client, err) + } + log.Println(resp) +} + +// printFeatures lists all the features within the given bounding Rectangle. +func build(client pb.BuilderClient, in *pb.BuildParams) { + stream, err := client.Build(context.Background(), in) + if err != nil { + log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + } + for { + line, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + } + log.Println(line) + } +} + +func main() { + conn, err := grpc.Dial("localhost:12345", grpc.WithInsecure()) + if err != nil { + log.Fatalf("fail to dial: %v", err) + } + defer conn.Close() + + client := pb.NewBuilderClient(conn) + + exampleParames := pb.BuildParams{ + BuiltInLibrariesFolders: "/ssd/Arduino-master/build/linux/work/libraries", + CustomBuildProperties: "build.warn_data_percentage=75", + FQBN: "arduino:avr:mega:cpu=atmega2560", + HardwareFolders: "/ssd/Arduino-master/build/linux/work/hardware,/home/martino/.arduino15/packages,/home/martino/eslov-sk/hardware", + OtherLibrariesFolders: "/home/martino/eslov-sk/libraries", + ArduinoAPIVersion: "10805", + SketchLocation: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino", + ToolsFolders: "/ssd/Arduino-master/build/linux/work/tools-builder,/ssd/Arduino-master/build/linux/work/hardware/tools/avr,/home/martino/.arduino15/packages", + Verbose: true, + WarningsLevel: "all", + BuildCachePath: "/tmp/arduino_cache_761418/", + CodeCompleteAt: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino:1:1", + } + + build(client, &exampleParames) + //autocomplete(client, &exampleParames) +} diff --git a/grpc/proto/builder.pb.go b/grpc/proto/builder.pb.go new file mode 100644 index 00000000..83161152 --- /dev/null +++ b/grpc/proto/builder.pb.go @@ -0,0 +1,343 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: builder.proto + +/* +Package proto is a generated protocol buffer package. + +It is generated from these files: + builder.proto + +It has these top-level messages: + BuildParams + Response +*/ +package proto + +import proto1 "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto1.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto1.ProtoPackageIsVersion2 // please upgrade the proto package + +type BuildParams struct { + HardwareFolders string `protobuf:"bytes,1,opt,name=hardwareFolders" json:"hardwareFolders,omitempty"` + ToolsFolders string `protobuf:"bytes,2,opt,name=toolsFolders" json:"toolsFolders,omitempty"` + BuiltInLibrariesFolders string `protobuf:"bytes,3,opt,name=builtInLibrariesFolders" json:"builtInLibrariesFolders,omitempty"` + OtherLibrariesFolders string `protobuf:"bytes,4,opt,name=otherLibrariesFolders" json:"otherLibrariesFolders,omitempty"` + SketchLocation string `protobuf:"bytes,5,opt,name=sketchLocation" json:"sketchLocation,omitempty"` + FQBN string `protobuf:"bytes,6,opt,name=fQBN" json:"fQBN,omitempty"` + ArduinoAPIVersion string `protobuf:"bytes,7,opt,name=arduinoAPIVersion" json:"arduinoAPIVersion,omitempty"` + CustomBuildProperties string `protobuf:"bytes,8,opt,name=customBuildProperties" json:"customBuildProperties,omitempty"` + BuildCachePath string `protobuf:"bytes,9,opt,name=buildCachePath" json:"buildCachePath,omitempty"` + BuildPath string `protobuf:"bytes,10,opt,name=buildPath" json:"buildPath,omitempty"` + WarningsLevel string `protobuf:"bytes,11,opt,name=warningsLevel" json:"warningsLevel,omitempty"` + CodeCompleteAt string `protobuf:"bytes,12,opt,name=codeCompleteAt" json:"codeCompleteAt,omitempty"` + Verbose bool `protobuf:"varint,13,opt,name=verbose" json:"verbose,omitempty"` +} + +func (m *BuildParams) Reset() { *m = BuildParams{} } +func (m *BuildParams) String() string { return proto1.CompactTextString(m) } +func (*BuildParams) ProtoMessage() {} +func (*BuildParams) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *BuildParams) GetHardwareFolders() string { + if m != nil { + return m.HardwareFolders + } + return "" +} + +func (m *BuildParams) GetToolsFolders() string { + if m != nil { + return m.ToolsFolders + } + return "" +} + +func (m *BuildParams) GetBuiltInLibrariesFolders() string { + if m != nil { + return m.BuiltInLibrariesFolders + } + return "" +} + +func (m *BuildParams) GetOtherLibrariesFolders() string { + if m != nil { + return m.OtherLibrariesFolders + } + return "" +} + +func (m *BuildParams) GetSketchLocation() string { + if m != nil { + return m.SketchLocation + } + return "" +} + +func (m *BuildParams) GetFQBN() string { + if m != nil { + return m.FQBN + } + return "" +} + +func (m *BuildParams) GetArduinoAPIVersion() string { + if m != nil { + return m.ArduinoAPIVersion + } + return "" +} + +func (m *BuildParams) GetCustomBuildProperties() string { + if m != nil { + return m.CustomBuildProperties + } + return "" +} + +func (m *BuildParams) GetBuildCachePath() string { + if m != nil { + return m.BuildCachePath + } + return "" +} + +func (m *BuildParams) GetBuildPath() string { + if m != nil { + return m.BuildPath + } + return "" +} + +func (m *BuildParams) GetWarningsLevel() string { + if m != nil { + return m.WarningsLevel + } + return "" +} + +func (m *BuildParams) GetCodeCompleteAt() string { + if m != nil { + return m.CodeCompleteAt + } + return "" +} + +func (m *BuildParams) GetVerbose() bool { + if m != nil { + return m.Verbose + } + return false +} + +type Response struct { + Line string `protobuf:"bytes,1,opt,name=line" json:"line,omitempty"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto1.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *Response) GetLine() string { + if m != nil { + return m.Line + } + return "" +} + +func init() { + proto1.RegisterType((*BuildParams)(nil), "proto.BuildParams") + proto1.RegisterType((*Response)(nil), "proto.Response") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Builder service + +type BuilderClient interface { + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + Build(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (Builder_BuildClient, error) + Autocomplete(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (*Response, error) +} + +type builderClient struct { + cc *grpc.ClientConn +} + +func NewBuilderClient(cc *grpc.ClientConn) BuilderClient { + return &builderClient{cc} +} + +func (c *builderClient) Build(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (Builder_BuildClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Builder_serviceDesc.Streams[0], c.cc, "/proto.Builder/Build", opts...) + if err != nil { + return nil, err + } + x := &builderBuildClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Builder_BuildClient interface { + Recv() (*Response, error) + grpc.ClientStream +} + +type builderBuildClient struct { + grpc.ClientStream +} + +func (x *builderBuildClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *builderClient) Autocomplete(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := grpc.Invoke(ctx, "/proto.Builder/Autocomplete", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Builder service + +type BuilderServer interface { + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + Build(*BuildParams, Builder_BuildServer) error + Autocomplete(context.Context, *BuildParams) (*Response, error) +} + +func RegisterBuilderServer(s *grpc.Server, srv BuilderServer) { + s.RegisterService(&_Builder_serviceDesc, srv) +} + +func _Builder_Build_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(BuildParams) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BuilderServer).Build(m, &builderBuildServer{stream}) +} + +type Builder_BuildServer interface { + Send(*Response) error + grpc.ServerStream +} + +type builderBuildServer struct { + grpc.ServerStream +} + +func (x *builderBuildServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func _Builder_Autocomplete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BuildParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BuilderServer).Autocomplete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Builder/Autocomplete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BuilderServer).Autocomplete(ctx, req.(*BuildParams)) + } + return interceptor(ctx, in, info, handler) +} + +var _Builder_serviceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Builder", + HandlerType: (*BuilderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Autocomplete", + Handler: _Builder_Autocomplete_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Build", + Handler: _Builder_Build_Handler, + ServerStreams: true, + }, + }, + Metadata: "builder.proto", +} + +func init() { proto1.RegisterFile("builder.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 389 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xdf, 0x8b, 0xd3, 0x40, + 0x10, 0xc7, 0x8d, 0xb6, 0xd7, 0x76, 0x2e, 0xf5, 0x70, 0x40, 0x5c, 0x44, 0xe4, 0x08, 0x87, 0xf4, + 0x41, 0xc2, 0xe1, 0x0f, 0xf0, 0xb5, 0x39, 0x10, 0x0e, 0x8a, 0xc4, 0x3e, 0xf8, 0xbe, 0xd9, 0x8c, + 0x66, 0x31, 0xcd, 0x84, 0xdd, 0xcd, 0xdd, 0xdf, 0xe2, 0x7f, 0x2b, 0xbb, 0x49, 0xd5, 0xb4, 0x15, + 0xee, 0xa9, 0xd3, 0xef, 0xf7, 0x33, 0x99, 0x2f, 0x3b, 0x03, 0xcb, 0xa2, 0xd3, 0x75, 0x49, 0x26, + 0x6d, 0x0d, 0x3b, 0xc6, 0x69, 0xf8, 0x49, 0x7e, 0x4d, 0xe0, 0x3c, 0xf3, 0x46, 0x2e, 0x8d, 0xdc, + 0x59, 0x5c, 0xc1, 0x45, 0x25, 0x4d, 0x79, 0x2f, 0x0d, 0x7d, 0x66, 0x8f, 0x5b, 0x11, 0x5d, 0x46, + 0xab, 0xc5, 0xf6, 0x50, 0xc6, 0x04, 0x62, 0xc7, 0x5c, 0xdb, 0x3d, 0xf6, 0x38, 0x60, 0x23, 0x0d, + 0x3f, 0xc1, 0x0b, 0x3f, 0xd5, 0xdd, 0x36, 0x1b, 0x5d, 0x18, 0x69, 0x34, 0xfd, 0xc1, 0x9f, 0x04, + 0xfc, 0x7f, 0x36, 0x7e, 0x80, 0xe7, 0xec, 0x2a, 0x32, 0x47, 0x7d, 0x93, 0xd0, 0x77, 0xda, 0xc4, + 0x37, 0xf0, 0xd4, 0xfe, 0x24, 0xa7, 0xaa, 0x0d, 0x2b, 0xe9, 0x34, 0x37, 0x62, 0x1a, 0xf0, 0x03, + 0x15, 0x11, 0x26, 0xdf, 0xbf, 0x66, 0x5f, 0xc4, 0x59, 0x70, 0x43, 0x8d, 0x6f, 0xe1, 0x99, 0x34, + 0x65, 0xa7, 0x1b, 0x5e, 0xe7, 0xb7, 0xdf, 0xc8, 0x58, 0xdf, 0x3e, 0x0b, 0xc0, 0xb1, 0xe1, 0xf3, + 0xa9, 0xce, 0x3a, 0xde, 0xf5, 0x8f, 0x67, 0xb8, 0x25, 0xe3, 0x34, 0x59, 0x31, 0xef, 0xf3, 0x9d, + 0x34, 0x7d, 0xbe, 0xb0, 0x85, 0x1b, 0xa9, 0x2a, 0xca, 0xa5, 0xab, 0xc4, 0xa2, 0xcf, 0x37, 0x56, + 0xf1, 0x15, 0x2c, 0x8a, 0x7e, 0x29, 0xae, 0x12, 0x10, 0x90, 0xbf, 0x02, 0x5e, 0xc1, 0xf2, 0x5e, + 0x9a, 0x46, 0x37, 0x3f, 0xec, 0x86, 0xee, 0xa8, 0x16, 0xe7, 0x81, 0x18, 0x8b, 0x7e, 0x96, 0xe2, + 0x92, 0x6e, 0x78, 0xd7, 0xd6, 0xe4, 0x68, 0xed, 0x44, 0xdc, 0xcf, 0x1a, 0xab, 0x28, 0x60, 0x76, + 0x47, 0xa6, 0x60, 0x4b, 0x62, 0x79, 0x19, 0xad, 0xe6, 0xdb, 0xfd, 0xdf, 0xe4, 0x35, 0xcc, 0xb7, + 0x64, 0x5b, 0x6e, 0x2c, 0xf9, 0x17, 0xab, 0x75, 0x43, 0xc3, 0x31, 0x84, 0xfa, 0x9d, 0x81, 0x59, + 0xd6, 0xdf, 0x14, 0x5e, 0xc3, 0x34, 0x94, 0x88, 0xfd, 0x79, 0xa5, 0xff, 0xdc, 0xd4, 0xcb, 0x8b, + 0x41, 0xdb, 0x7f, 0x2c, 0x79, 0x74, 0x1d, 0xe1, 0x47, 0x88, 0xd7, 0x9d, 0x63, 0x35, 0x04, 0x79, + 0x60, 0x63, 0x76, 0x05, 0xa8, 0x54, 0x3a, 0xec, 0x23, 0x1d, 0x4e, 0x3a, 0x8b, 0x87, 0x1c, 0xb9, + 0xc7, 0xf3, 0xa8, 0x38, 0x0b, 0x7d, 0xef, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x37, 0x9b, + 0x2d, 0xf4, 0x02, 0x00, 0x00, +} diff --git a/grpc/proto/builder.proto b/grpc/proto/builder.proto new file mode 100644 index 00000000..1e9a7892 --- /dev/null +++ b/grpc/proto/builder.proto @@ -0,0 +1,57 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// compile me with: protoc -I proto/ proto/builder.proto --go_out=plugins=grpc:proto + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "cc.arduino.builder"; +option java_outer_classname = "BuilderProto"; + +package proto; + +// Interface exported by the server. +service Builder { + + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + rpc Build(BuildParams) returns (stream Response) {} + + rpc Autocomplete(BuildParams) returns (Response) {} +} + +message BuildParams { + string hardwareFolders = 1; + string toolsFolders = 2; + string builtInLibrariesFolders = 3; + string otherLibrariesFolders = 4; + string sketchLocation = 5; + string fQBN = 6; + string arduinoAPIVersion = 7; + string customBuildProperties = 8; + string buildCachePath = 9; + string buildPath = 10; + string warningsLevel = 11; + string codeCompleteAt = 12; + bool verbose = 13; +} + +message Response { + string line = 1; +} diff --git a/grpc/rpc.go b/grpc/rpc.go new file mode 100644 index 00000000..12ec4c5a --- /dev/null +++ b/grpc/rpc.go @@ -0,0 +1,180 @@ +package jsonrpc + +import ( + "fmt" + "io" + "log" + "net" + "strings" + + builder "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/types" + "github.com/fsnotify/fsnotify" + "golang.org/x/net/context" + "google.golang.org/grpc" + + pb "github.com/arduino/arduino-builder/grpc/proto" +) + +type StreamLogger struct { + stream pb.Builder_BuildServer +} + +func (s StreamLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + s.stream.Send(&pb.Response{Line: fmt.Sprintf(format, a...)}) +} + +func (s StreamLogger) UnformattedFprintln(w io.Writer, str string) { + s.stream.Send(&pb.Response{Line: str}) +} + +func (s StreamLogger) UnformattedWrite(w io.Writer, data []byte) { + s.stream.Send(&pb.Response{Line: string(data)}) +} + +func (s StreamLogger) Println(level string, format string, a ...interface{}) { + s.stream.Send(&pb.Response{Line: fmt.Sprintf(format, a...)}) +} + +func (s StreamLogger) Flush() string { + return "" +} + +func (s StreamLogger) Name() string { + return "streamlogger" +} + +type builderServer struct { + resp []*pb.Response + ctx *types.Context + watcher *fsnotify.Watcher +} + +// GetFeature returns the feature at the given point. +func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) (*pb.Response, error) { + + s.ctx.HardwareFolders = strings.Split(args.HardwareFolders, ",") + s.ctx.ToolsFolders = strings.Split(args.ToolsFolders, ",") + s.ctx.BuiltInLibrariesFolders = strings.Split(args.BuiltInLibrariesFolders, ",") + s.ctx.OtherLibrariesFolders = strings.Split(args.OtherLibrariesFolders, ",") + s.ctx.SketchLocation = args.SketchLocation + s.ctx.CustomBuildProperties = strings.Split(args.CustomBuildProperties, ",") + s.ctx.ArduinoAPIVersion = args.ArduinoAPIVersion + s.ctx.FQBN = args.FQBN + s.ctx.Verbose = false //p.Verbose + s.ctx.BuildCachePath = args.BuildCachePath + s.ctx.BuildPath = args.BuildPath + s.ctx.WarningsLevel = args.WarningsLevel + s.ctx.PrototypesSection = "" + s.ctx.CodeCompleteAt = args.CodeCompleteAt + + s.ctx.IncludeFolders = s.ctx.IncludeFolders[0:0] + s.ctx.LibrariesObjectFiles = s.ctx.LibrariesObjectFiles[0:0] + s.ctx.CoreObjectsFiles = s.ctx.CoreObjectsFiles[0:0] + s.ctx.SketchObjectFiles = s.ctx.SketchObjectFiles[0:0] + + s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] + + err := builder.RunPreprocess(s.ctx) + if err != nil { + return &pb.Response{Line: s.ctx.GetLogger().Flush()}, err + } + + // No feature was found, return an unnamed feature + return &pb.Response{Line: s.ctx.GetLogger().Flush()}, nil +} + +// GetFeature returns the feature at the given point. +func (s *builderServer) Build(args *pb.BuildParams, stream pb.Builder_BuildServer) error { + + s.ctx.HardwareFolders = strings.Split(args.HardwareFolders, ",") + s.ctx.ToolsFolders = strings.Split(args.ToolsFolders, ",") + s.ctx.BuiltInLibrariesFolders = strings.Split(args.BuiltInLibrariesFolders, ",") + s.ctx.OtherLibrariesFolders = strings.Split(args.OtherLibrariesFolders, ",") + s.ctx.SketchLocation = args.SketchLocation + s.ctx.CustomBuildProperties = strings.Split(args.CustomBuildProperties, ",") + s.ctx.ArduinoAPIVersion = args.ArduinoAPIVersion + s.ctx.FQBN = args.FQBN + s.ctx.Verbose = args.Verbose + s.ctx.BuildCachePath = args.BuildCachePath + s.ctx.BuildPath = args.BuildPath + s.ctx.WarningsLevel = args.WarningsLevel + s.ctx.PrototypesSection = "" + s.ctx.CodeCompleteAt = "" + + s.ctx.IncludeFolders = s.ctx.IncludeFolders[0:0] + s.ctx.LibrariesObjectFiles = s.ctx.LibrariesObjectFiles[0:0] + s.ctx.CoreObjectsFiles = s.ctx.CoreObjectsFiles[0:0] + s.ctx.SketchObjectFiles = s.ctx.SketchObjectFiles[0:0] + + s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] + + // setup logger to send via protobuf + oldlogger := s.ctx.GetLogger() + logger := StreamLogger{stream} + s.ctx.SetLogger(logger) + + err := builder.RunBuilder(s.ctx) + s.ctx.SetLogger(oldlogger) + if err != nil { + return err + } + + // No feature was found, return an unnamed feature + return nil +} + +/* +func (h *WatchHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { + + var p WatchParams + if err := jsonrpc.Unmarshal(params, &p); err != nil { + return nil, err + } + + err := h.watcher.Add(p.Path) + if err != nil { + return nil, jsonrpc.ErrInvalidParams() + } + return BuildResult{ + Message: "OK " + p.Path, + }, nil +} +*/ + +func startWatching(ctx *types.Context) *fsnotify.Watcher { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + go func() { + for { + select { + case event := <-watcher.Events: + ctx.CanUseCachedTools = false + log.Println("event:", event) + case err := <-watcher.Errors: + log.Println("error:", err) + } + } + }() + return watcher +} + +func newServer(ctx *types.Context, watcher *fsnotify.Watcher) *builderServer { + s := new(builderServer) + s.ctx = ctx + s.watcher = watcher + return s +} + +func RegisterAndServeJsonRPC(ctx *types.Context) { + + watcher := startWatching(ctx) + + lis, _ := net.Listen("tcp", "localhost:12345") + grpcServer := grpc.NewServer() + pb.RegisterBuilderServer(grpcServer, newServer(ctx, watcher)) + grpcServer.Serve(lis) +} diff --git a/jsonrpc/rpc.go b/jsonrpc/rpc.go deleted file mode 100644 index 3992f388..00000000 --- a/jsonrpc/rpc.go +++ /dev/null @@ -1,216 +0,0 @@ -package jsonrpc - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "strings" - - builder "github.com/arduino/arduino-builder" - "github.com/arduino/arduino-builder/types" - "github.com/fsnotify/fsnotify" - "github.com/osamingo/jsonrpc" - "golang.org/x/net/context" -) - -type ( - BuildHandler struct { - watcher *fsnotify.Watcher - ctx *types.Context - } - BuildParams struct { - HardwareFolders string - ToolsFolders string - BuiltInLibrariesFolders string - OtherLibrariesFolders string - SketchLocation string - FQBN string - ArduinoAPIVersion string - CustomBuildProperties string - Verbose bool - BuildCachePath string - BuildPath string - WarningsLevel string - } - BuildResult struct { - Message string `json:"message"` - Error error - } -) - -type ( - CompleteHandler struct { - watcher *fsnotify.Watcher - ctx *types.Context - } - CompleteParams struct { - HardwareFolders string - ToolsFolders string - BuiltInLibrariesFolders string - OtherLibrariesFolders string - SketchLocation string - FQBN string - ArduinoAPIVersion string - CustomBuildProperties string - Verbose bool - BuildCachePath string - BuildPath string - WarningsLevel string - CodeCompleteAt string - } - CompleteResult struct { - Message string `json:"message"` - Error error - } -) - -type ( - WatchHandler struct { - watcher *fsnotify.Watcher - } - WatchParams struct { - Path string `json:"path"` - } - WatchResult struct { - Message string `json:"message"` - } -) - -func (h *CompleteHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { - - var p CompleteParams - if err := jsonrpc.Unmarshal(params, &p); err != nil { - fmt.Println(err) - return nil, err - } - - h.ctx.HardwareFolders = strings.Split(p.HardwareFolders, ",") - h.ctx.ToolsFolders = strings.Split(p.ToolsFolders, ",") - h.ctx.BuiltInLibrariesFolders = strings.Split(p.BuiltInLibrariesFolders, ",") - h.ctx.OtherLibrariesFolders = strings.Split(p.OtherLibrariesFolders, ",") - h.ctx.SketchLocation = p.SketchLocation - h.ctx.CustomBuildProperties = strings.Split(p.CustomBuildProperties, ",") - h.ctx.ArduinoAPIVersion = p.ArduinoAPIVersion - h.ctx.FQBN = p.FQBN - h.ctx.Verbose = false //p.Verbose - h.ctx.BuildCachePath = p.BuildCachePath - h.ctx.BuildPath = p.BuildPath - h.ctx.WarningsLevel = p.WarningsLevel - h.ctx.PrototypesSection = "" - h.ctx.CodeCompleteAt = p.CodeCompleteAt - - h.ctx.IncludeFolders = h.ctx.IncludeFolders[0:0] - h.ctx.LibrariesObjectFiles = h.ctx.LibrariesObjectFiles[0:0] - h.ctx.CoreObjectsFiles = h.ctx.CoreObjectsFiles[0:0] - h.ctx.SketchObjectFiles = h.ctx.SketchObjectFiles[0:0] - - h.ctx.ImportedLibraries = h.ctx.ImportedLibraries[0:0] - - err := builder.RunPreprocess(h.ctx) - if err != nil { - return BuildResult{ - Message: h.ctx.GetLogger().Flush(), - Error: err, - }, nil - } - - return CompleteResult{ - Message: h.ctx.GetLogger().Flush(), - }, nil -} - -func (h *BuildHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { - - var p BuildParams - if err := jsonrpc.Unmarshal(params, &p); err != nil { - fmt.Println(err) - return nil, err - } - - h.ctx.HardwareFolders = strings.Split(p.HardwareFolders, ",") - h.ctx.ToolsFolders = strings.Split(p.ToolsFolders, ",") - h.ctx.BuiltInLibrariesFolders = strings.Split(p.BuiltInLibrariesFolders, ",") - h.ctx.OtherLibrariesFolders = strings.Split(p.OtherLibrariesFolders, ",") - h.ctx.SketchLocation = p.SketchLocation - h.ctx.CustomBuildProperties = strings.Split(p.CustomBuildProperties, ",") - h.ctx.ArduinoAPIVersion = p.ArduinoAPIVersion - h.ctx.FQBN = p.FQBN - h.ctx.Verbose = p.Verbose - h.ctx.BuildCachePath = p.BuildCachePath - h.ctx.BuildPath = p.BuildPath - h.ctx.WarningsLevel = p.WarningsLevel - h.ctx.PrototypesSection = "" - - h.ctx.IncludeFolders = h.ctx.IncludeFolders[0:0] - h.ctx.LibrariesObjectFiles = h.ctx.LibrariesObjectFiles[0:0] - h.ctx.CoreObjectsFiles = h.ctx.CoreObjectsFiles[0:0] - h.ctx.SketchObjectFiles = h.ctx.SketchObjectFiles[0:0] - - h.ctx.ImportedLibraries = h.ctx.ImportedLibraries[0:0] - - err := builder.RunBuilder(h.ctx) - if err != nil { - return BuildResult{ - Message: h.ctx.GetLogger().Flush(), - Error: err, - }, nil - } - - return BuildResult{ - Message: h.ctx.GetLogger().Flush(), - }, nil -} - -func (h *WatchHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { - - var p WatchParams - if err := jsonrpc.Unmarshal(params, &p); err != nil { - return nil, err - } - - err := h.watcher.Add(p.Path) - if err != nil { - return nil, jsonrpc.ErrInvalidParams() - } - return BuildResult{ - Message: "OK " + p.Path, - }, nil -} - -func startWatching(ctx *types.Context) *fsnotify.Watcher { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - - go func() { - for { - select { - case event := <-watcher.Events: - ctx.CanUseCachedTools = false - log.Println("event:", event) - case err := <-watcher.Errors: - log.Println("error:", err) - } - } - }() - return watcher -} - -func RegisterAndServeJsonRPC(ctx *types.Context) { - - watcher := startWatching(ctx) - - jsonrpc.RegisterMethod("Main.Build", &BuildHandler{watcher, ctx}, BuildParams{}, BuildResult{}) - jsonrpc.RegisterMethod("Main.Complete", &CompleteHandler{watcher, ctx}, CompleteParams{}, CompleteResult{}) - jsonrpc.RegisterMethod("Main.AddWatchPath", &WatchHandler{watcher}, WatchParams{}, WatchResult{}) - - http.HandleFunc("/jrpc", func(w http.ResponseWriter, r *http.Request) { - jsonrpc.HandlerFunc(r.Context(), w, r) - }) - http.HandleFunc("/jrpc/debug", jsonrpc.DebugHandlerFunc) - if err := http.ListenAndServe(":8888", nil); err != nil { - log.Fatalln(err) - } -} From 758c3eceebeb70f708c9fa5fb714cddba601ae11 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 19 Sep 2017 17:03:11 +0200 Subject: [PATCH 14/68] add working watcher implementation --- client/client.go | 6 +++--- grpc/rpc.go | 23 +++++++++++++++++++++++ types/context.go | 1 + utils/utils.go | 24 ++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index 9576073d..11184387 100644 --- a/client/client.go +++ b/client/client.go @@ -79,9 +79,9 @@ func main() { Verbose: true, WarningsLevel: "all", BuildCachePath: "/tmp/arduino_cache_761418/", - CodeCompleteAt: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino:1:1", + CodeCompleteAt: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino:56:9", } - build(client, &exampleParames) - //autocomplete(client, &exampleParames) + //build(client, &exampleParames) + autocomplete(client, &exampleParames) } diff --git a/grpc/rpc.go b/grpc/rpc.go index 12ec4c5a..3c726ffb 100644 --- a/grpc/rpc.go +++ b/grpc/rpc.go @@ -9,6 +9,7 @@ import ( builder "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" "github.com/fsnotify/fsnotify" "golang.org/x/net/context" "google.golang.org/grpc" @@ -50,6 +51,24 @@ type builderServer struct { watcher *fsnotify.Watcher } +func (s *builderServer) watch() { + folders := [][]string{s.ctx.HardwareFolders, s.ctx.ToolsFolders, s.ctx.BuiltInLibrariesFolders, s.ctx.OtherLibrariesFolders} + + for _, category := range folders { + for _, folder := range category { + if !utils.SliceContains(s.ctx.WatchedLocations, folder) { + var subfolders []string + utils.FindAllSubdirectories(folder, &subfolders) + subfolders = append(subfolders, folder) + for _, element := range subfolders { + s.watcher.Add(element) + s.ctx.WatchedLocations = utils.AppendIfNotPresent(s.ctx.WatchedLocations, element) + } + } + } + } +} + // GetFeature returns the feature at the given point. func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) (*pb.Response, error) { @@ -75,6 +94,8 @@ func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] + s.watch() + err := builder.RunPreprocess(s.ctx) if err != nil { return &pb.Response{Line: s.ctx.GetLogger().Flush()}, err @@ -114,6 +135,8 @@ func (s *builderServer) Build(args *pb.BuildParams, stream pb.Builder_BuildServe logger := StreamLogger{stream} s.ctx.SetLogger(logger) + s.watch() + err := builder.RunBuilder(s.ctx) s.ctx.SetLogger(oldlogger) if err != nil { diff --git a/types/context.go b/types/context.go index 5801844f..f880d142 100644 --- a/types/context.go +++ b/types/context.go @@ -15,6 +15,7 @@ type Context struct { LibrariesFolders []string BuiltInLibrariesFolders []string OtherLibrariesFolders []string + WatchedLocations []string SketchLocation string ArduinoAPIVersion string FQBN string diff --git a/utils/utils.go b/utils/utils.go index a5682c29..2ffb3aca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -325,6 +325,30 @@ func FilterOutFoldersByNames(folders []os.FileInfo, names ...string) []os.FileIn type CheckExtensionFunc func(ext string) bool +func FindAllSubdirectories(folder string, output *[]string) error { + walkFunc := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip source control and hidden files and directories + if IsSCCSOrHiddenFile(info) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Skip directories unless recurse is on, or this is the + // root directory + if info.IsDir() { + *output = AppendIfNotPresent(*output, path) + } + return nil + } + return gohasissues.Walk(folder, walkFunc) +} + func FindFilesInFolder(files *[]string, folder string, extensions CheckExtensionFunc, recurse bool) error { walkFunc := func(path string, info os.FileInfo, err error) error { if err != nil { From ea2e91bb2afb4abee66aea09bca0c5eda0d5dbe8 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 25 Sep 2017 15:15:09 +0200 Subject: [PATCH 15/68] Use relative paths if commandline is too long --- builder_utils/utils.go | 10 +++++++++- ctags_runner.go | 2 +- preprocess_sketch.go | 2 +- utils/utils.go | 25 +++++++++++++++++++++---- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index b89346e4..a0fc0a2b 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -418,6 +418,8 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties return bytes, i18n.WrapError(err) } +const COMMANDLINE_LIMIT = 32000 + func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) { pattern := buildProperties[recipe] if pattern == constants.EMPTY_STRING { @@ -430,7 +432,13 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) } - command, err := utils.PrepareCommand(commandLine, logger) + relativePath := "" + + if len(commandLine) > COMMANDLINE_LIMIT { + relativePath = buildProperties[constants.BUILD_PROPERTIES_BUILD_PATH] + } + + command, err := utils.PrepareCommand(commandLine, logger, relativePath) if err != nil { return nil, i18n.WrapError(err) } diff --git a/ctags_runner.go b/ctags_runner.go index 86bb3cfb..ef4dc64a 100644 --- a/ctags_runner.go +++ b/ctags_runner.go @@ -56,7 +56,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { } commandLine := properties.ExpandPropsInString(pattern) - command, err := utils.PrepareCommand(commandLine, logger) + command, err := utils.PrepareCommand(commandLine, logger, "") if err != nil { return i18n.WrapError(err) } diff --git a/preprocess_sketch.go b/preprocess_sketch.go index ee459939..fb967a04 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -88,7 +88,7 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { } commandLine := properties.ExpandPropsInString(pattern) - command, err := utils.PrepareCommand(commandLine, logger) + command, err := utils.PrepareCommand(commandLine, logger, "") if err != nil { return i18n.WrapError(err) } diff --git a/utils/utils.go b/utils/utils.go index 2ffb3aca..24e90b6c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -231,7 +231,7 @@ func TrimSpace(value string) string { type argFilterFunc func(int, string, []string) bool -func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18n.Logger) (*exec.Cmd, error) { +func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18n.Logger, relativePath string) (*exec.Cmd, error) { parts, err := ParseCommandLine(pattern, logger) if err != nil { return nil, i18n.WrapError(err) @@ -241,19 +241,36 @@ func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18 var args []string for idx, part := range parts { if filter(idx, part, parts) { + // if relativePath is specified, the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + if relativePath != "" { + if _, err := os.Stat(part); !os.IsNotExist(err) { + tmp, err := filepath.Rel(relativePath, part) + if err == nil { + part = tmp + } + } + } args = append(args, part) } } - return exec.Command(command, args...), nil + cmd := exec.Command(command, args...) + + if relativePath != "" { + cmd.Dir = relativePath + } + + return cmd, nil } func filterEmptyArg(_ int, arg string, _ []string) bool { return arg != constants.EMPTY_STRING } -func PrepareCommand(pattern string, logger i18n.Logger) (*exec.Cmd, error) { - return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger) +func PrepareCommand(pattern string, logger i18n.Logger, relativePath string) (*exec.Cmd, error) { + return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger, relativePath) } func MapHas(aMap map[string]interface{}, key string) bool { From 361953fac31dd39fc84ec4458e0fcd97dfa324be Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 5 Oct 2017 18:07:35 +0200 Subject: [PATCH 16/68] Don't try spawining more than one daemonized builder --- grpc/rpc.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/grpc/rpc.go b/grpc/rpc.go index 3c726ffb..18e4d762 100644 --- a/grpc/rpc.go +++ b/grpc/rpc.go @@ -5,6 +5,7 @@ import ( "io" "log" "net" + "os" "strings" builder "github.com/arduino/arduino-builder" @@ -193,10 +194,13 @@ func newServer(ctx *types.Context, watcher *fsnotify.Watcher) *builderServer { } func RegisterAndServeJsonRPC(ctx *types.Context) { - + lis, err := net.Listen("tcp", "localhost:12345") + if err != nil { + //can't spawn two grpc servers on the same port + os.Exit(0) + } watcher := startWatching(ctx) - lis, _ := net.Listen("tcp", "localhost:12345") grpcServer := grpc.NewServer() pb.RegisterBuilderServer(grpcServer, newServer(ctx, watcher)) grpcServer.Serve(lis) From e33005255192221bcf959e641d9d87957c976cd2 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 5 Oct 2017 18:36:26 +0200 Subject: [PATCH 17/68] update to arduino-preprocessor 0.1.3 --- test/helper_tools_downloader.go | 12 ++++++------ test/tools_loader_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index 3b422339..c327c991 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -114,13 +114,13 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }, }, - Tool{Name: "arduino-preprocessor", Version: "0.1.1", + Tool{Name: "arduino-preprocessor", Version: "0.1.3", OsUrls: []OsUrl{ - OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-i686-pc-cygwin.tar.bz2"}, - OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-x86_64-apple-darwin11.tar.bz2"}, - OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.1/arduino-preprocessor-0.1.1-armhf-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-pc-cygwin-2.tar.bz2"}, + OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-apple-darwin11.tar.bz2"}, + OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-armhf-pc-linux-gnu.tar.bz2"}, }, }, } diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index 46036507..48afa6dd 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -71,8 +71,8 @@ func TestLoadTools(t *testing.T) { idx := 0 require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.1", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder) + require.Equal(t, "0.1.3", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.3"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) From 253195c4d4af1461c70cd1047238b5d54e3ddd7a Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 5 Oct 2017 18:38:18 +0200 Subject: [PATCH 18/68] remove debug print --- tools_loader.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools_loader.go b/tools_loader.go index d4f9c233..8fd3241d 100644 --- a/tools_loader.go +++ b/tools_loader.go @@ -30,7 +30,6 @@ package builder import ( - "fmt" "os" "path/filepath" "strings" @@ -50,7 +49,7 @@ func (s *ToolsLoader) Run(ctx *types.Context) error { tools := []*types.Tool{} if ctx.CanUseCachedTools { - fmt.Println("no fs modification, can use cached ctx") + //fmt.Println("no fs modification, can use cached ctx") return nil } From db0055f5bb88dad62d8b2494b4f625143ef4dad8 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 6 Oct 2017 19:10:00 +0200 Subject: [PATCH 19/68] Don't try to absolutize the filepath if empty On Windows, it fails with the rather cryptic error message "The filename, directory name, or volume label syntax is incorrect" --- utils/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 24e90b6c..bfb92a5b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -293,6 +293,9 @@ func SliceToMapStringBool(keys []string, value bool) map[string]bool { func AbsolutizePaths(files []string) ([]string, error) { for idx, file := range files { + if file == "" { + continue + } absFile, err := filepath.Abs(file) if err != nil { return nil, i18n.WrapError(err) From 52e78fc5f6fb1046b3d6f0c84b65fa995adfddd4 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 9 Oct 2017 12:33:46 +0200 Subject: [PATCH 20/68] [WIN] make filepath relative to overcome parsing difficulties Fixing https://github.com/arduino/arduino-preprocessor/blob/master/CommandLine.cpp#L91 should have the same effect but let's try with this first --- preprocess_sketch.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/preprocess_sketch.go b/preprocess_sketch.go index fb967a04..11493144 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -32,6 +32,8 @@ package builder import ( "fmt" "path/filepath" + "runtime" + "strings" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" @@ -77,6 +79,16 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { properties.Merge(toolProps) properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = targetFilePath if ctx.CodeCompleteAt != "" { + if runtime.GOOS == "windows" { + //use relative filepath to avoid ":" escaping + splt := strings.Split(ctx.CodeCompleteAt, ":") + if len(splt) == 3 { + //all right, do nothing + } else { + splt[1] = filepath.Base(splt[0] + ":" + splt[1]) + ctx.CodeCompleteAt = strings.Join(splt[1:], ":") + } + } properties["codecomplete"] = "-output-code-completions=" + ctx.CodeCompleteAt } else { properties["codecomplete"] = "" From 44e6d735fc4ab377854f3f54df42a352c0af6156 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 9 Oct 2017 12:37:50 +0200 Subject: [PATCH 21/68] [WIN] USe cross-compiled arduino-preprocessor --- test/helper_tools_downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index c327c991..556f126c 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -118,7 +118,7 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrls: []OsUrl{ OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-pc-linux-gnu.tar.bz2"}, OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-pc-cygwin-2.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-w64-mingw32.tar.bz2"}, OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-apple-darwin11.tar.bz2"}, OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-armhf-pc-linux-gnu.tar.bz2"}, }, From b4be062596e3604140651e92a978a156ec128034 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 12 Oct 2017 17:51:05 +0200 Subject: [PATCH 22/68] Mute log output during autocomplete via rpc Should solve https://github.com/arduino/Arduino/issues/6818 --- grpc/rpc.go | 14 +++++++++----- preprocess_sketch.go | 6 +++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/grpc/rpc.go b/grpc/rpc.go index 18e4d762..f2bf0c4d 100644 --- a/grpc/rpc.go +++ b/grpc/rpc.go @@ -9,6 +9,7 @@ import ( "strings" builder "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/fsnotify/fsnotify" @@ -87,6 +88,7 @@ func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) s.ctx.WarningsLevel = args.WarningsLevel s.ctx.PrototypesSection = "" s.ctx.CodeCompleteAt = args.CodeCompleteAt + s.ctx.CodeCompletions = "" s.ctx.IncludeFolders = s.ctx.IncludeFolders[0:0] s.ctx.LibrariesObjectFiles = s.ctx.LibrariesObjectFiles[0:0] @@ -96,14 +98,16 @@ func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] s.watch() + oldlogger := s.ctx.GetLogger() + logger := i18n.NoopLogger{} + s.ctx.SetLogger(logger) err := builder.RunPreprocess(s.ctx) - if err != nil { - return &pb.Response{Line: s.ctx.GetLogger().Flush()}, err - } - // No feature was found, return an unnamed feature - return &pb.Response{Line: s.ctx.GetLogger().Flush()}, nil + response := pb.Response{Line: s.ctx.CodeCompletions} + s.ctx.SetLogger(oldlogger) + + return &response, err } // GetFeature returns the feature at the given point. diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 11493144..8ea4e786 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -127,6 +127,10 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { type OutputCodeCompletions struct{} func (s *OutputCodeCompletions) Run(ctx *types.Context) error { - ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, "%s", ctx.CodeCompletions) + if ctx.CodeCompletions == "" { + // we assume it is a json, let's make it compliant at least + ctx.CodeCompletions = "[]" + } + ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, ctx.CodeCompletions) return nil } From dcce66b19ec9363f0b4f6ea7fee726d90b6741e7 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 13 Oct 2017 16:05:24 +0200 Subject: [PATCH 23/68] Add NormalizeUTF8 utility function --- utils/utils.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index bfb92a5b..3d26f971 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -40,12 +40,15 @@ import ( "path/filepath" "runtime" "strings" + "unicode" "unicode/utf8" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/gohasissues" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" ) func KeysOfMapOfStringInterface(input map[string]interface{}) []string { @@ -535,6 +538,18 @@ func ParseCppString(line string) (string, string, bool) { } } +func isMn(r rune) bool { + return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks +} + +// Normalizes an UTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + // CopyFile copies the contents of the file named src to the file named // by dst. The file will be created if it does not already exist. If the // destination file exists, all it's contents will be replaced by the contents From 7de76a97a24bfa6ebc8a08f85121915ad99473d6 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 13 Oct 2017 16:05:49 +0200 Subject: [PATCH 24/68] [WIN] Launch arduino-preprocessor from volume name path This workarounds https://github.com/arduino/arduino-preprocessor/issues/2 --- preprocess_sketch.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 8ea4e786..9caa0797 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -105,6 +105,12 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { return i18n.WrapError(err) } + if runtime.GOOS == "windows" { + // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) + command.Dir = filepath.VolumeName(command.Args[0]) + "/" + //command.Args[0], _ = filepath.Rel(command.Dir, command.Args[0]) + } + verbose := ctx.Verbose if verbose { fmt.Println(commandLine) From 0dae510263b30f988dc50f7828e1283437f86d91 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 13 Oct 2017 16:08:02 +0200 Subject: [PATCH 25/68] Mute ctx.SourceGccMinusE if empty --- print_preprocessed_source.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/print_preprocessed_source.go b/print_preprocessed_source.go index c6afae14..b320c127 100644 --- a/print_preprocessed_source.go +++ b/print_preprocessed_source.go @@ -30,13 +30,16 @@ package builder import ( - "github.com/arduino/arduino-builder/types" "fmt" + + "github.com/arduino/arduino-builder/types" ) type PrintPreprocessedSource struct{} func (s *PrintPreprocessedSource) Run(ctx *types.Context) error { - fmt.Println(ctx.SourceGccMinusE) + if ctx.SourceGccMinusE != "" { + fmt.Println(ctx.SourceGccMinusE) + } return nil } From ae6b63578d9eaa8efbf9067d9239596446418247 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Mon, 16 Oct 2017 19:10:13 +0200 Subject: [PATCH 26/68] Normalize preprocessed output Solves autocomplete over grcp Fixes arduino/Arduino#6816 --- preprocess_sketch.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 9caa0797..a5f25691 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -30,7 +30,9 @@ package builder import ( + "errors" "fmt" + "os/exec" "path/filepath" "runtime" "strings" @@ -118,14 +120,16 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { buf, err := command.Output() if err != nil { - return i18n.WrapError(err) + return errors.New(i18n.WrapError(err).Error() + string(err.(*exec.ExitError).Stderr)) } - output := string(buf) + + result := utils.NormalizeUTF8(buf) + //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) if ctx.CodeCompleteAt != "" { - ctx.CodeCompletions = output + ctx.CodeCompletions = string(result) } else { - ctx.Source = output + ctx.Source = string(result) } return nil } From d9f55b4b2ba547fb3caeb71037a3d8ec8df42023 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 18 Oct 2017 16:57:13 +0200 Subject: [PATCH 27/68] Remove watcher, add endpoint to explicitely drop cache --- grpc/proto/builder.pb.go | 107 +++++++++++++++++++++++++++++---------- grpc/proto/builder.proto | 6 +++ grpc/rpc.go | 23 +++++++-- 3 files changed, 104 insertions(+), 32 deletions(-) diff --git a/grpc/proto/builder.pb.go b/grpc/proto/builder.pb.go index 83161152..4690c81c 100644 --- a/grpc/proto/builder.pb.go +++ b/grpc/proto/builder.pb.go @@ -9,6 +9,7 @@ It is generated from these files: It has these top-level messages: BuildParams + VerboseParams Response */ package proto @@ -145,6 +146,22 @@ func (m *BuildParams) GetVerbose() bool { return false } +type VerboseParams struct { + Verbose bool `protobuf:"varint,1,opt,name=verbose" json:"verbose,omitempty"` +} + +func (m *VerboseParams) Reset() { *m = VerboseParams{} } +func (m *VerboseParams) String() string { return proto1.CompactTextString(m) } +func (*VerboseParams) ProtoMessage() {} +func (*VerboseParams) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *VerboseParams) GetVerbose() bool { + if m != nil { + return m.Verbose + } + return false +} + type Response struct { Line string `protobuf:"bytes,1,opt,name=line" json:"line,omitempty"` } @@ -152,7 +169,7 @@ type Response struct { func (m *Response) Reset() { *m = Response{} } func (m *Response) String() string { return proto1.CompactTextString(m) } func (*Response) ProtoMessage() {} -func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } func (m *Response) GetLine() string { if m != nil { @@ -163,6 +180,7 @@ func (m *Response) GetLine() string { func init() { proto1.RegisterType((*BuildParams)(nil), "proto.BuildParams") + proto1.RegisterType((*VerboseParams)(nil), "proto.VerboseParams") proto1.RegisterType((*Response)(nil), "proto.Response") } @@ -185,6 +203,7 @@ type BuilderClient interface { // huge number of features. Build(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (Builder_BuildClient, error) Autocomplete(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (*Response, error) + DropCache(ctx context.Context, in *VerboseParams, opts ...grpc.CallOption) (*Response, error) } type builderClient struct { @@ -236,6 +255,15 @@ func (c *builderClient) Autocomplete(ctx context.Context, in *BuildParams, opts return out, nil } +func (c *builderClient) DropCache(ctx context.Context, in *VerboseParams, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := grpc.Invoke(ctx, "/proto.Builder/DropCache", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for Builder service type BuilderServer interface { @@ -247,6 +275,7 @@ type BuilderServer interface { // huge number of features. Build(*BuildParams, Builder_BuildServer) error Autocomplete(context.Context, *BuildParams) (*Response, error) + DropCache(context.Context, *VerboseParams) (*Response, error) } func RegisterBuilderServer(s *grpc.Server, srv BuilderServer) { @@ -292,6 +321,24 @@ func _Builder_Autocomplete_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _Builder_DropCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VerboseParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BuilderServer).DropCache(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Builder/DropCache", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BuilderServer).DropCache(ctx, req.(*VerboseParams)) + } + return interceptor(ctx, in, info, handler) +} + var _Builder_serviceDesc = grpc.ServiceDesc{ ServiceName: "proto.Builder", HandlerType: (*BuilderServer)(nil), @@ -300,6 +347,10 @@ var _Builder_serviceDesc = grpc.ServiceDesc{ MethodName: "Autocomplete", Handler: _Builder_Autocomplete_Handler, }, + { + MethodName: "DropCache", + Handler: _Builder_DropCache_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -314,30 +365,32 @@ var _Builder_serviceDesc = grpc.ServiceDesc{ func init() { proto1.RegisterFile("builder.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 389 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xdf, 0x8b, 0xd3, 0x40, - 0x10, 0xc7, 0x8d, 0xb6, 0xd7, 0x76, 0x2e, 0xf5, 0x70, 0x40, 0x5c, 0x44, 0xe4, 0x08, 0x87, 0xf4, - 0x41, 0xc2, 0xe1, 0x0f, 0xf0, 0xb5, 0x39, 0x10, 0x0e, 0x8a, 0xc4, 0x3e, 0xf8, 0xbe, 0xd9, 0x8c, - 0x66, 0x31, 0xcd, 0x84, 0xdd, 0xcd, 0xdd, 0xdf, 0xe2, 0x7f, 0x2b, 0xbb, 0x49, 0xd5, 0xb4, 0x15, - 0xee, 0xa9, 0xd3, 0xef, 0xf7, 0x33, 0x99, 0x2f, 0x3b, 0x03, 0xcb, 0xa2, 0xd3, 0x75, 0x49, 0x26, - 0x6d, 0x0d, 0x3b, 0xc6, 0x69, 0xf8, 0x49, 0x7e, 0x4d, 0xe0, 0x3c, 0xf3, 0x46, 0x2e, 0x8d, 0xdc, - 0x59, 0x5c, 0xc1, 0x45, 0x25, 0x4d, 0x79, 0x2f, 0x0d, 0x7d, 0x66, 0x8f, 0x5b, 0x11, 0x5d, 0x46, - 0xab, 0xc5, 0xf6, 0x50, 0xc6, 0x04, 0x62, 0xc7, 0x5c, 0xdb, 0x3d, 0xf6, 0x38, 0x60, 0x23, 0x0d, - 0x3f, 0xc1, 0x0b, 0x3f, 0xd5, 0xdd, 0x36, 0x1b, 0x5d, 0x18, 0x69, 0x34, 0xfd, 0xc1, 0x9f, 0x04, - 0xfc, 0x7f, 0x36, 0x7e, 0x80, 0xe7, 0xec, 0x2a, 0x32, 0x47, 0x7d, 0x93, 0xd0, 0x77, 0xda, 0xc4, - 0x37, 0xf0, 0xd4, 0xfe, 0x24, 0xa7, 0xaa, 0x0d, 0x2b, 0xe9, 0x34, 0x37, 0x62, 0x1a, 0xf0, 0x03, - 0x15, 0x11, 0x26, 0xdf, 0xbf, 0x66, 0x5f, 0xc4, 0x59, 0x70, 0x43, 0x8d, 0x6f, 0xe1, 0x99, 0x34, - 0x65, 0xa7, 0x1b, 0x5e, 0xe7, 0xb7, 0xdf, 0xc8, 0x58, 0xdf, 0x3e, 0x0b, 0xc0, 0xb1, 0xe1, 0xf3, - 0xa9, 0xce, 0x3a, 0xde, 0xf5, 0x8f, 0x67, 0xb8, 0x25, 0xe3, 0x34, 0x59, 0x31, 0xef, 0xf3, 0x9d, - 0x34, 0x7d, 0xbe, 0xb0, 0x85, 0x1b, 0xa9, 0x2a, 0xca, 0xa5, 0xab, 0xc4, 0xa2, 0xcf, 0x37, 0x56, - 0xf1, 0x15, 0x2c, 0x8a, 0x7e, 0x29, 0xae, 0x12, 0x10, 0x90, 0xbf, 0x02, 0x5e, 0xc1, 0xf2, 0x5e, - 0x9a, 0x46, 0x37, 0x3f, 0xec, 0x86, 0xee, 0xa8, 0x16, 0xe7, 0x81, 0x18, 0x8b, 0x7e, 0x96, 0xe2, - 0x92, 0x6e, 0x78, 0xd7, 0xd6, 0xe4, 0x68, 0xed, 0x44, 0xdc, 0xcf, 0x1a, 0xab, 0x28, 0x60, 0x76, - 0x47, 0xa6, 0x60, 0x4b, 0x62, 0x79, 0x19, 0xad, 0xe6, 0xdb, 0xfd, 0xdf, 0xe4, 0x35, 0xcc, 0xb7, - 0x64, 0x5b, 0x6e, 0x2c, 0xf9, 0x17, 0xab, 0x75, 0x43, 0xc3, 0x31, 0x84, 0xfa, 0x9d, 0x81, 0x59, - 0xd6, 0xdf, 0x14, 0x5e, 0xc3, 0x34, 0x94, 0x88, 0xfd, 0x79, 0xa5, 0xff, 0xdc, 0xd4, 0xcb, 0x8b, - 0x41, 0xdb, 0x7f, 0x2c, 0x79, 0x74, 0x1d, 0xe1, 0x47, 0x88, 0xd7, 0x9d, 0x63, 0x35, 0x04, 0x79, - 0x60, 0x63, 0x76, 0x05, 0xa8, 0x54, 0x3a, 0xec, 0x23, 0x1d, 0x4e, 0x3a, 0x8b, 0x87, 0x1c, 0xb9, - 0xc7, 0xf3, 0xa8, 0x38, 0x0b, 0x7d, 0xef, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x37, 0x9b, - 0x2d, 0xf4, 0x02, 0x00, 0x00, + // 420 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x93, 0x6f, 0x6b, 0x13, 0x41, + 0x10, 0xc6, 0x3d, 0x4d, 0x9a, 0x64, 0x9a, 0x58, 0x1c, 0x14, 0x17, 0x11, 0x29, 0xa1, 0x48, 0x04, + 0x09, 0x45, 0x2b, 0xf8, 0x36, 0x57, 0x11, 0x0a, 0x41, 0xce, 0xbc, 0xe8, 0xfb, 0xbd, 0xcd, 0xe8, + 0x2d, 0x5e, 0x6e, 0x8e, 0xdd, 0x4d, 0xfb, 0x59, 0xfc, 0x06, 0x7e, 0x4c, 0xd9, 0x3f, 0xd1, 0x5e, + 0x13, 0xc1, 0x57, 0x99, 0x3c, 0xcf, 0x6f, 0x6e, 0x9f, 0xdb, 0x99, 0x83, 0x49, 0xb9, 0xd5, 0xf5, + 0x9a, 0xcc, 0xbc, 0x35, 0xec, 0x18, 0xfb, 0xe1, 0x67, 0xfa, 0xb3, 0x07, 0xc7, 0xb9, 0x37, 0x0a, + 0x69, 0xe4, 0xc6, 0xe2, 0x0c, 0x4e, 0x2a, 0x69, 0xd6, 0xb7, 0xd2, 0xd0, 0x67, 0xf6, 0xb8, 0x15, + 0xd9, 0x69, 0x36, 0x1b, 0xad, 0xee, 0xcb, 0x38, 0x85, 0xb1, 0x63, 0xae, 0xed, 0x0e, 0x7b, 0x18, + 0xb0, 0x8e, 0x86, 0x1f, 0xe1, 0xb9, 0x3f, 0xd5, 0x5d, 0x35, 0x4b, 0x5d, 0x1a, 0x69, 0x34, 0xfd, + 0xc1, 0x1f, 0x05, 0xfc, 0x5f, 0x36, 0x5e, 0xc0, 0x33, 0x76, 0x15, 0x99, 0xbd, 0xbe, 0x5e, 0xe8, + 0x3b, 0x6c, 0xe2, 0x6b, 0x78, 0x6c, 0x7f, 0x90, 0x53, 0xd5, 0x92, 0x95, 0x74, 0x9a, 0x1b, 0xd1, + 0x0f, 0xf8, 0x3d, 0x15, 0x11, 0x7a, 0xdf, 0xbe, 0xe6, 0x5f, 0xc4, 0x51, 0x70, 0x43, 0x8d, 0x6f, + 0xe1, 0x89, 0x34, 0xeb, 0xad, 0x6e, 0x78, 0x51, 0x5c, 0x5d, 0x93, 0xb1, 0xbe, 0x7d, 0x10, 0x80, + 0x7d, 0xc3, 0xe7, 0x53, 0x5b, 0xeb, 0x78, 0x13, 0x2f, 0xcf, 0x70, 0x4b, 0xc6, 0x69, 0xb2, 0x62, + 0x18, 0xf3, 0x1d, 0x34, 0x7d, 0xbe, 0x30, 0x85, 0x4b, 0xa9, 0x2a, 0x2a, 0xa4, 0xab, 0xc4, 0x28, + 0xe6, 0xeb, 0xaa, 0xf8, 0x12, 0x46, 0x65, 0x1c, 0x8a, 0xab, 0x04, 0x04, 0xe4, 0xaf, 0x80, 0x67, + 0x30, 0xb9, 0x95, 0xa6, 0xd1, 0xcd, 0x77, 0xbb, 0xa4, 0x1b, 0xaa, 0xc5, 0x71, 0x20, 0xba, 0xa2, + 0x3f, 0x4b, 0xf1, 0x9a, 0x2e, 0x79, 0xd3, 0xd6, 0xe4, 0x68, 0xe1, 0xc4, 0x38, 0x9e, 0xd5, 0x55, + 0x51, 0xc0, 0xe0, 0x86, 0x4c, 0xc9, 0x96, 0xc4, 0xe4, 0x34, 0x9b, 0x0d, 0x57, 0xbb, 0xbf, 0xd3, + 0x37, 0x30, 0xb9, 0x8e, 0x65, 0x5a, 0x8e, 0x3b, 0x68, 0xd6, 0x45, 0x5f, 0xc1, 0x70, 0x45, 0xb6, + 0xe5, 0xc6, 0x92, 0xbf, 0xdc, 0x5a, 0x37, 0x94, 0xf6, 0x26, 0xd4, 0xef, 0x7e, 0x65, 0x30, 0xc8, + 0xe3, 0xfe, 0xe1, 0x39, 0xf4, 0x43, 0x89, 0x18, 0x57, 0x71, 0x7e, 0x67, 0xff, 0x5e, 0x9c, 0x24, + 0x6d, 0xf7, 0xb4, 0xe9, 0x83, 0xf3, 0x0c, 0x3f, 0xc0, 0x78, 0xb1, 0x75, 0xac, 0x52, 0xe8, 0xff, + 0x6c, 0xc4, 0x0b, 0x18, 0x7d, 0x32, 0xdc, 0x86, 0x6b, 0xc5, 0xa7, 0xc9, 0xef, 0xbc, 0xd1, 0x81, + 0xae, 0xfc, 0x0c, 0x50, 0xa9, 0x79, 0x9a, 0xf8, 0x3c, 0x7d, 0x34, 0xf9, 0x38, 0xa5, 0x2f, 0x3c, + 0x5e, 0x64, 0xe5, 0x51, 0xe8, 0x7b, 0xff, 0x3b, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x39, 0x66, 0x51, + 0x56, 0x03, 0x00, 0x00, } diff --git a/grpc/proto/builder.proto b/grpc/proto/builder.proto index 1e9a7892..9b310284 100644 --- a/grpc/proto/builder.proto +++ b/grpc/proto/builder.proto @@ -34,6 +34,8 @@ service Builder { rpc Build(BuildParams) returns (stream Response) {} rpc Autocomplete(BuildParams) returns (Response) {} + + rpc DropCache(VerboseParams) returns (Response) {} } message BuildParams { @@ -52,6 +54,10 @@ message BuildParams { bool verbose = 13; } +message VerboseParams { + bool verbose = 1; +} + message Response { string line = 1; } diff --git a/grpc/rpc.go b/grpc/rpc.go index f2bf0c4d..55cd78ff 100644 --- a/grpc/rpc.go +++ b/grpc/rpc.go @@ -71,6 +71,12 @@ func (s *builderServer) watch() { } } +func (s *builderServer) DropCache(ctx context.Context, args *pb.VerboseParams) (*pb.Response, error) { + s.ctx.CanUseCachedTools = false + response := pb.Response{Line: "Tools cache dropped"} + return &response, nil +} + // GetFeature returns the feature at the given point. func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) (*pb.Response, error) { @@ -97,7 +103,8 @@ func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] - s.watch() + //s.watch() + oldlogger := s.ctx.GetLogger() logger := i18n.NoopLogger{} s.ctx.SetLogger(logger) @@ -140,7 +147,7 @@ func (s *builderServer) Build(args *pb.BuildParams, stream pb.Builder_BuildServe logger := StreamLogger{stream} s.ctx.SetLogger(logger) - s.watch() + //s.watch() err := builder.RunBuilder(s.ctx) s.ctx.SetLogger(oldlogger) @@ -190,22 +197,28 @@ func startWatching(ctx *types.Context) *fsnotify.Watcher { return watcher } -func newServer(ctx *types.Context, watcher *fsnotify.Watcher) *builderServer { +func newServerWithWatcher(ctx *types.Context, watcher *fsnotify.Watcher) *builderServer { s := new(builderServer) s.ctx = ctx s.watcher = watcher return s } +func newServer(ctx *types.Context) *builderServer { + s := new(builderServer) + s.ctx = ctx + return s +} + func RegisterAndServeJsonRPC(ctx *types.Context) { lis, err := net.Listen("tcp", "localhost:12345") if err != nil { //can't spawn two grpc servers on the same port os.Exit(0) } - watcher := startWatching(ctx) + //watcher := startWatching(ctx) grpcServer := grpc.NewServer() - pb.RegisterBuilderServer(grpcServer, newServer(ctx, watcher)) + pb.RegisterBuilderServer(grpcServer, newServer(ctx)) grpcServer.Serve(lis) } From de716ee2f636c45d220a8ed2e5572404418d65f5 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 19 Oct 2017 17:32:08 +0200 Subject: [PATCH 28/68] Fix cmake generation for difficult targets --- create_cmake_rule.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/create_cmake_rule.go b/create_cmake_rule.go index 085efb91..c9b23c6f 100644 --- a/create_cmake_rule.go +++ b/create_cmake_rule.go @@ -85,6 +85,16 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { if _, err := os.Stat(filepath.Join(libFolder, "examples")); err == nil { os.RemoveAll(filepath.Join(libFolder, "examples")) } + // Remove stray folders contining incompatible libraries + staticLibsExtensions := func(ext string) bool { return DOTAEXTENSION[ext] } + mcu := ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_MCU] + var files []string + utils.FindFilesInFolder(&files, filepath.Join(libFolder, "src"), staticLibsExtensions, true) + for _, file := range files { + if !strings.Contains(filepath.Dir(file), mcu) { + os.RemoveAll(filepath.Dir(file)) + } + } } // Copy core + variant in use + preprocessed sketch in the correct folders @@ -171,7 +181,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { // Compile and link project cmakelist += "add_executable (" + projectName + " ${SOURCES} ${SOURCES_LIBS})\n" - cmakelist += "target_link_libraries( " + projectName + " " + strings.Join(libs, " ") + ")\n" + cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed " + strings.Join(libs, " ") + ")\n" utils.WriteFile(cmakeFile, cmakelist) @@ -198,7 +208,7 @@ func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, link *linkDirectories = appendIfUnique(*linkDirectories, strings.TrimPrefix(arg, "-L")) continue } - if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") { + if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") && !strings.HasPrefix(arg, "-o") { // HACK : from linkerflags remove MMD (no cache is produced) if !strings.HasPrefix(arg, "-MMD") { *linkerflags = appendIfUnique(*linkerflags, arg) From e2e9471fa85d83a8f405e224b4b5239691df7dcf Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 20 Oct 2017 09:42:08 +0200 Subject: [PATCH 29/68] Lower max commandline length bound --- builder_utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index a0fc0a2b..0ec2aaba 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -418,7 +418,7 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties return bytes, i18n.WrapError(err) } -const COMMANDLINE_LIMIT = 32000 +const COMMANDLINE_LIMIT = 30000 func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) { pattern := buildProperties[recipe] From a888c62f594ba1aa1820bac186f196732e4de8f8 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 31 Oct 2017 14:19:27 +0100 Subject: [PATCH 30/68] [CMAKE] export only preprocessed source --- create_cmake_rule.go | 13 +++++++++++++ filter_sketch_source.go | 8 ++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/create_cmake_rule.go b/create_cmake_rule.go index c9b23c6f..a37d1cc6 100644 --- a/create_cmake_rule.go +++ b/create_cmake_rule.go @@ -106,6 +106,19 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { if err != nil { fmt.Println(err) } + + // Use old ctags method to generate export file + commands := []types.Command{ + &ContainerMergeCopySketchFiles{}, + &ContainerAddPrototypes{}, + &FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true}, + &SketchSaver{}, + } + + for _, command := range commands { + command.Run(ctx) + } + err = utils.CopyDir(ctx.SketchBuildPath, filepath.Join(cmakeFolder, "sketch"), extensions) if err != nil { fmt.Println(err) diff --git a/filter_sketch_source.go b/filter_sketch_source.go index 6de658cd..a6abd730 100644 --- a/filter_sketch_source.go +++ b/filter_sketch_source.go @@ -39,7 +39,8 @@ import ( ) type FilterSketchSource struct { - Source *string + Source *string + RemoveLineMarkers bool } func (s *FilterSketchSource) Run(ctx *types.Context) error { @@ -57,6 +58,9 @@ func (s *FilterSketchSource) Run(ctx *types.Context) error { filename := parseLineMarker(line) if filename != "" { inSketch = utils.SliceContains(fileNames, filename) + if inSketch && s.RemoveLineMarkers { + continue + } } if inSketch { @@ -79,7 +83,7 @@ func parseLineMarker(line string) string { // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 split := strings.SplitN(line, " ", 3) - if len(split) < 3 || split[0] != "#" { + if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { return "" } From 15baf786d982da4f52bd26f1fdd186fd6e66a893 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 31 Oct 2017 14:30:55 +0100 Subject: [PATCH 31/68] [CMAKE] use single group for all libraries --- create_cmake_rule.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_cmake_rule.go b/create_cmake_rule.go index a37d1cc6..5166447b 100644 --- a/create_cmake_rule.go +++ b/create_cmake_rule.go @@ -194,7 +194,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { // Compile and link project cmakelist += "add_executable (" + projectName + " ${SOURCES} ${SOURCES_LIBS})\n" - cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed " + strings.Join(libs, " ") + ")\n" + cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed -Wl,--start-group " + strings.Join(libs, " ") + " -Wl,--end-group)\n" utils.WriteFile(cmakeFile, cmakelist) From e7600b54770fe353094090049dd87f6b9fa91032 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 31 Oct 2017 14:53:31 +0100 Subject: [PATCH 32/68] [CMAKE] include Arduino.h in main sketch file --- sketch_source_merger.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sketch_source_merger.go b/sketch_source_merger.go index bbfaad0d..f2c91cb6 100644 --- a/sketch_source_merger.go +++ b/sketch_source_merger.go @@ -31,9 +31,10 @@ package builder import ( + "regexp" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "regexp" ) type SketchSourceMerger struct{} @@ -43,12 +44,12 @@ func (s *SketchSourceMerger) Run(ctx *types.Context) error { lineOffset := 0 includeSection := "" + includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n" + lineOffset++ if !sketchIncludesArduinoH(&sketch.MainFile) { includeSection += "#include \n" lineOffset++ } - includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n" - lineOffset++ ctx.IncludeSection = includeSection source := includeSection From 140e24fdfeca84b6aafb20f9321305b0f47033c1 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 12:55:56 +0200 Subject: [PATCH 33/68] Fix past-end-of-cache handling in includeCache.ExpectFile The comments state that if ExpectFile() is called, but there are no remaining items in the cache, it will invalidate the cache. However, the code would only invalidate the cache if at least one item was still present, but it didn't match the expected file. In practice, this wouldn't usually cause issues, since adding a new file would usually cause an invalid cache earlier on, and even if a new file was added at the end of the compilation, it would not be in the .d file, so it would be marked as "changed". However, in rare circumstances, such as when the include cache would not be properly generated due to other problems (see #230), this would cause a crash, when ExpectFile did not invalidate the cache and the file in question was unchanged, causing an out-of-bounds read from the cache. This commit fixes this by making ExpectFile behave like documented and invalidate the cache when there are no remaining entries. Signed-off-by: Matthijs Kooijman --- container_find_includes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container_find_includes.go b/container_find_includes.go index dbd31c71..a06cf059 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -213,7 +213,7 @@ func (cache *includeCache) Next() includeCacheEntry { // not, or no entry is available, the cache is invalidated. Does not // advance the cache. func (cache *includeCache) ExpectFile(sourcefile string) { - if cache.valid && cache.next < len(cache.entries) && cache.Next().Sourcefile != sourcefile { + if cache.valid && (cache.next >= len(cache.entries) || cache.Next().Sourcefile != sourcefile) { cache.valid = false cache.entries = cache.entries[:cache.next] } From 85c5781a6c8b5316537002a7a7eb1f2ad33eacc2 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 18:10:26 +0200 Subject: [PATCH 34/68] Convert IncludesFinderWithRegExp to a normal function Signed-off-by: Matthijs Kooijman --- container_find_includes.go | 3 +- includes_finder_with_regexp.go | 14 ++----- test/includes_finder_with_regexp_test.go | 50 ++++++------------------ types/context.go | 1 - 4 files changed, 16 insertions(+), 52 deletions(-) diff --git a/container_find_includes.go b/container_find_includes.go index a06cf059..c5feed38 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -325,7 +325,6 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t } else { commands := []types.Command{ &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes}, - &IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE}, } for _, command := range commands { err := runCommand(ctx, command) @@ -333,7 +332,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t return i18n.WrapError(err) } } - include = ctx.IncludeJustFound + include = IncludesFinderWithRegExp(ctx, ctx.SourceGccMinusE) } if include == "" { diff --git a/includes_finder_with_regexp.go b/includes_finder_with_regexp.go index 56ac955e..abfe7635 100644 --- a/includes_finder_with_regexp.go +++ b/includes_finder_with_regexp.go @@ -37,21 +37,13 @@ import ( var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") -type IncludesFinderWithRegExp struct { - Source *string -} - -func (s *IncludesFinderWithRegExp) Run(ctx *types.Context) error { - source := *s.Source - +func IncludesFinderWithRegExp(ctx *types.Context, source string) string { match := INCLUDE_REGEXP.FindStringSubmatch(source) if match != nil { - ctx.IncludeJustFound = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } else { - ctx.IncludeJustFound = findIncludeForOldCompilers(source) + return findIncludeForOldCompilers(source) } - - return nil } func findIncludeForOldCompilers(source string) string { diff --git a/test/includes_finder_with_regexp_test.go b/test/includes_finder_with_regexp_test.go index d95153a5..c3c048cc 100644 --- a/test/includes_finder_with_regexp_test.go +++ b/test/includes_finder_with_regexp_test.go @@ -43,27 +43,17 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { ctx := &types.Context{} - output := "" - - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, "") - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "", ctx.IncludeJustFound) + require.Equal(t, "", include) } func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { @@ -73,13 +63,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { @@ -89,13 +75,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { @@ -104,13 +86,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { @@ -119,11 +97,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "register.h", ctx.IncludeJustFound) + require.Equal(t, "register.h", include) } diff --git a/types/context.go b/types/context.go index f880d142..a9c5b857 100644 --- a/types/context.go +++ b/types/context.go @@ -64,7 +64,6 @@ type Context struct { HeaderToLibraries map[string][]*Library ImportedLibraries []*Library LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeJustFound string IncludeFolders []string OutputGccMinusM string From 39e3c8413b910c49a0c1dffea8689c4198bb2394 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 19:07:40 +0200 Subject: [PATCH 35/68] Convert GCCPreprocRunner(ForDiscoveringIncludes) to a normal function Signed-off-by: Matthijs Kooijman --- container_add_prototypes.go | 7 ++++++- container_find_includes.go | 15 +++++---------- gcc_preproc_runner.go | 28 +++++++--------------------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/container_add_prototypes.go b/container_add_prototypes.go index cd0d56f7..14f30199 100644 --- a/container_add_prototypes.go +++ b/container_add_prototypes.go @@ -41,8 +41,13 @@ type ContainerAddPrototypes struct{} func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + + // Run preprocessor + err := GCCPreprocRunner(ctx, sourceFile, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, ctx.IncludeFolders) + if err != nil { + return i18n.WrapError(err) + } commands := []types.Command{ - &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, &ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E}, diff --git a/container_find_includes.go b/container_find_includes.go index c5feed38..91bafd7c 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -323,16 +323,11 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_CACHED_INCLUDES, sourcePath) } } else { - commands := []types.Command{ - &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes}, + stderr, err := GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + if err != nil { + return i18n.WrapError(err) } - for _, command := range commands { - err := runCommand(ctx, command) - if err != nil { - return i18n.WrapError(err) - } - } - include = IncludesFinderWithRegExp(ctx, ctx.SourceGccMinusE) + include = IncludesFinderWithRegExp(ctx, stderr) } if include == "" { @@ -344,7 +339,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes}) + err := GCCPreprocRunner(ctx, sourcePath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, includes) return i18n.WrapError(err) } diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index b571cacc..9d3bd948 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -41,14 +41,8 @@ import ( "github.com/arduino/go-properties-map" ) -type GCCPreprocRunner struct { - SourceFilePath string - TargetFileName string - Includes []string -} - -func (s *GCCPreprocRunner) Run(ctx *types.Context) error { - properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes) +func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error { + properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return i18n.WrapError(err) } @@ -70,16 +64,10 @@ func (s *GCCPreprocRunner) Run(ctx *types.Context) error { return nil } -type GCCPreprocRunnerForDiscoveringIncludes struct { - SourceFilePath string - TargetFilePath string - Includes []string -} - -func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { - properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes) +func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (string, error) { + properties, _, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { - return i18n.WrapError(err) + return "", i18n.WrapError(err) } verbose := ctx.Verbose @@ -92,12 +80,10 @@ func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) if err != nil { - return i18n.WrapError(err) + return "", i18n.WrapError(err) } - ctx.SourceGccMinusE = string(stderr) - - return nil + return string(stderr), nil } func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { From 652f83053b5ee2df6241cfdd0b68d81e9bf954d5 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 19:27:56 +0200 Subject: [PATCH 36/68] Refactor path generation for ctags_target_for_gcc_minus_e.cpp Previously, the relative filename passed to GCCPreprocRunner() was made absolute by prepareGCCPreprocRecipeProperties() and then returned, so it could be used later on. There was an exception to this when /dev/null was passed. Now, the only place that passed something other than /dev/null simply does this processing up front instead. This prepares the way for removing Context::FileToRead in the next commit. Signed-off-by: Matthijs Kooijman --- container_add_prototypes.go | 10 +++++++++- gcc_preproc_runner.go | 18 ++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/container_add_prototypes.go b/container_add_prototypes.go index 14f30199..fb1f04ec 100644 --- a/container_add_prototypes.go +++ b/container_add_prototypes.go @@ -35,6 +35,7 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" ) type ContainerAddPrototypes struct{} @@ -42,8 +43,15 @@ type ContainerAddPrototypes struct{} func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + // Generate the full pathname for the preproc output file + err := utils.EnsureFolderExists(ctx.PreprocPath) + if err != nil { + return i18n.WrapError(err) + } + targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E) + // Run preprocessor - err := GCCPreprocRunner(ctx, sourceFile, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, ctx.IncludeFolders) + err = GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) if err != nil { return i18n.WrapError(err) } diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 9d3bd948..6df8f421 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -30,7 +30,6 @@ package builder import ( - "path/filepath" "strings" "github.com/arduino/arduino-builder/builder_utils" @@ -42,7 +41,7 @@ import ( ) func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error { - properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) + properties, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return i18n.WrapError(err) } @@ -65,7 +64,7 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath } func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (string, error) { - properties, _, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) + properties, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return "", i18n.WrapError(err) } @@ -86,16 +85,7 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s return string(stderr), nil } -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { - if targetFilePath != utils.NULLFile() { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return nil, "", i18n.WrapError(err) - } - targetFilePath = filepath.Join(preprocPath, targetFilePath) - } - +func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, error) { properties := ctx.BuildProperties.Clone() properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath @@ -104,7 +94,7 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties) - return properties, targetFilePath, nil + return properties, nil } func GeneratePreprocPatternFromCompile(compilePattern string) string { From edff07dd1f3eb01f5ac32f75d20e694e2ad567be Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 19:33:44 +0200 Subject: [PATCH 37/68] Pass FileToRead to ReadFileAndStoreInContext explicitly Previously, this filename was set by GCCPreprocRunner into the context, because the full filename was not known until then. Since the previous commit, this filename is known by the ContainerAddPrototypes, which can just pass it to ReadFileAndStoreInContext explicitly. This allows Context::FileToRead to be removed. Signed-off-by: Matthijs Kooijman --- container_add_prototypes.go | 2 +- gcc_preproc_runner.go | 2 -- read_file_and_store_in_context.go | 3 ++- test/read_file_and_store_in_context_test.go | 3 +-- types/context.go | 3 --- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/container_add_prototypes.go b/container_add_prototypes.go index fb1f04ec..13b777d3 100644 --- a/container_add_prototypes.go +++ b/container_add_prototypes.go @@ -56,7 +56,7 @@ func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { return i18n.WrapError(err) } commands := []types.Command{ - &ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}, + &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E}, &CTagsRunner{}, diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 6df8f421..0f95917f 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -58,8 +58,6 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath return i18n.WrapError(err) } - ctx.FileToRead = targetFilePath - return nil } diff --git a/read_file_and_store_in_context.go b/read_file_and_store_in_context.go index a2533929..bd165617 100644 --- a/read_file_and_store_in_context.go +++ b/read_file_and_store_in_context.go @@ -36,11 +36,12 @@ import ( ) type ReadFileAndStoreInContext struct { + FileToRead string Target *string } func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error { - bytes, err := ioutil.ReadFile(ctx.FileToRead) + bytes, err := ioutil.ReadFile(s.FileToRead) if err != nil { return i18n.WrapError(err) } diff --git a/test/read_file_and_store_in_context_test.go b/test/read_file_and_store_in_context_test.go index a83cfa0c..b33ff47c 100644 --- a/test/read_file_and_store_in_context_test.go +++ b/test/read_file_and_store_in_context_test.go @@ -47,9 +47,8 @@ func TestReadFileAndStoreInContext(t *testing.T) { utils.WriteFile(file.Name(), "test test\nciao") ctx := &types.Context{} - ctx.FileToRead = file.Name() - command := &builder.ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE} + command := &builder.ReadFileAndStoreInContext{FileToRead: file.Name(), Target: &ctx.SourceGccMinusE} err = command.Run(ctx) NoError(t, err) diff --git a/types/context.go b/types/context.go index a9c5b857..cf3c026e 100644 --- a/types/context.go +++ b/types/context.go @@ -88,9 +88,6 @@ type Context struct { logger i18n.Logger DebugLevel int - // ReadFileAndStoreInContext command - FileToRead string - // Reuse old tools since the backing storage didn't change CanUseCachedTools bool } From b6b8a83349d02a8492046a275d9caed0d1e4b7c8 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 6 Jun 2017 19:35:35 +0200 Subject: [PATCH 38/68] Remove GCCPreprocSourceSaver This pass was unused. Signed-off-by: Matthijs Kooijman --- gcc_preproc_source_saver.go | 53 ------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 gcc_preproc_source_saver.go diff --git a/gcc_preproc_source_saver.go b/gcc_preproc_source_saver.go deleted file mode 100644 index 556cd276..00000000 --- a/gcc_preproc_source_saver.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This file is part of Arduino Builder. - * - * Arduino Builder is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - * As a special exception, you may use this file as part of a free software - * library without restriction. Specifically, if other files instantiate - * templates or use macros or inline functions from this file, or you compile - * this file and link it with other files to produce an executable, this - * file does not by itself cause the resulting executable to be covered by - * the GNU General Public License. This exception does not however - * invalidate any other reasons why the executable file might be covered by - * the GNU General Public License. - * - * Copyright 2015 Arduino LLC (http://www.arduino.cc/) - */ - -package builder - -// XXX: OBSOLETE? - -import ( - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" - "path/filepath" -) - -type GCCPreprocSourceSaver struct{} - -func (s *GCCPreprocSourceSaver) Run(ctx *types.Context) error { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return i18n.WrapError(err) - } - - err = utils.WriteFile(filepath.Join(preprocPath, constants.FILE_GCC_PREPROC_TARGET), ctx.Source) - return i18n.WrapError(err) -} From e56a5f5791280e5a6ee6fec2e08614e6d871d2bd Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 13:00:13 +0200 Subject: [PATCH 39/68] execSizeRecipe: Fix typo in method name Signed-off-by: Matthijs Kooijman --- phases/sizer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phases/sizer.go b/phases/sizer.go index 7ea86997..b1bedf75 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -89,7 +89,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin } } - textSize, dataSize, _, err := execSizeReceipe(properties, logger) + textSize, dataSize, _, err := execSizeRecipe(properties, logger) if err != nil { logger.Println(constants.LOG_LEVEL_WARN, constants.MSG_SIZER_ERROR_NO_RULE) return nil @@ -127,7 +127,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin return nil } -func execSizeReceipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) { +func execSizeRecipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) { out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, logger) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) From b3106c0657132a86488c76a2ce8fa5756e3d469b Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 13:18:55 +0200 Subject: [PATCH 40/68] Pass types.Context down into compilation helpers Previously, the verbose, logger and sometimes warningFlags were extracted from the Context by the top-level runner and passed down separately. Since this causes a lot of variable-passing of what is essentially global state, it is clearer to just pass a types.Context down and let the helpers get the data they need from that state. This prepared for a next commit where ExecRecipe will be refactored and needs access to the Context. Since the next commit will heavily change ExecRecipe anyway, this commit does not actually change ExecRecipe to accept the Context. This commit should not change any behaviour. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 38 +++++++++++++++++++------------------ phases/core_builder.go | 18 ++++++++---------- phases/libraries_builder.go | 22 ++++++++++----------- phases/linker.go | 11 ++++------- phases/sizer.go | 16 +++++++--------- phases/sketch_builder.go | 7 ++----- 6 files changed, 51 insertions(+), 61 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 0ec2aaba..6e083283 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -45,8 +45,8 @@ import ( "github.com/arduino/go-properties-map" ) -func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger) +func CompileFilesRecursive(ctx *types.Context, objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := CompileFiles(ctx, objectFiles, sourcePath, false, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -57,7 +57,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st } for _, folder := range folders { - objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = CompileFilesRecursive(ctx, objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -66,28 +66,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st return objectFiles, nil } -func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger) +func CompileFiles(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN) if err != nil { return nil, i18n.WrapError(err) } return objectFiles, nil } -func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithExtensionWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string) ([]string, error) { sources, err := findFilesInFolder(sourcePath, extension, recurse) if err != nil { return nil, i18n.WrapError(err) } - return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + return compileFilesWithRecipe(ctx, objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe) } func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) { @@ -146,7 +146,7 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { return sources, nil } -func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string) ([]string, error) { if len(sources) == 0 { return objectFiles, nil } @@ -160,7 +160,7 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s for _, source := range sources { go func(source string) { defer wg.Done() - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) if err != nil { errorsChan <- err } else { @@ -190,9 +190,10 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s } } -func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { +func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string) (string, error) { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = source relativeSource, err := filepath.Rel(sourcePath, source) @@ -212,11 +213,11 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b } if !objIsUpToDate { - _, err = ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, err = ExecRecipe(properties, recipe, false, ctx.Verbose, ctx.Verbose, logger) if err != nil { return "", i18n.WrapError(err) } - } else if verbose { + } else if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE]) } @@ -342,7 +343,8 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) return true } -func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { +func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map) (string, error) { + logger := ctx.GetLogger() archiveFilePath := filepath.Join(buildPath, archiveFile) rebuildArchive := false @@ -365,7 +367,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st return "", i18n.WrapError(err) } } else { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, archiveFilePath) } return archiveFilePath, nil @@ -378,7 +380,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = archiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile - _, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, verbose, verbose, logger) + _, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.Verbose, ctx.Verbose, logger) if err != nil { return "", i18n.WrapError(err) } diff --git a/phases/core_builder.go b/phases/core_builder.go index 75b6535d..cecea3de 100644 --- a/phases/core_builder.go +++ b/phases/core_builder.go @@ -46,9 +46,6 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { coreBuildPath := ctx.CoreBuildPath coreBuildCachePath := ctx.CoreBuildCachePath buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(coreBuildPath) if err != nil { @@ -62,7 +59,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { } } - archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger) + archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,8 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(ctx *types.Context, buildPath string, buildCachePath string, buildProperties properties.Map) (string, []string, error) { + logger := ctx.GetLogger() coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] @@ -90,7 +88,7 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper variantObjectFiles := []string{} if variantFolder != constants.EMPTY_STRING { - variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + variantObjectFiles, err = builder_utils.CompileFiles(ctx, variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } @@ -107,26 +105,26 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper if canUseArchivedCore { // use archived core - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core: {0}", targetArchivedCore) } return targetArchivedCore, variantObjectFiles, nil } } - coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + coreObjectFiles, err := builder_utils.CompileFiles(ctx, []string{}, coreFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } - archiveFile, err := builder_utils.ArchiveCompiledFiles(buildPath, "core.a", coreObjectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, "core.a", coreObjectFiles, buildProperties) if err != nil { return "", nil, i18n.WrapError(err) } // archive core.a if targetArchivedCore != "" { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_ARCHIVING_CORE_CACHE, targetArchivedCore) } builder_utils.CopyFile(archiveFile, targetArchivedCore) diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index d80fcf91..9cc1fecc 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -52,16 +52,13 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) libraries := ctx.ImportedLibraries - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(librariesBuildPath) if err != nil { return i18n.WrapError(err) } - objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err := compileLibraries(ctx, libraries, librariesBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -99,10 +96,10 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib return nil } -func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileLibraries(ctx *types.Context, libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { objectFiles := []string{} for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -113,8 +110,9 @@ func compileLibraries(libraries []*types.Library, buildPath string, buildPropert } -func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - if verbose { +func compileLibrary(ctx *types.Context, library *types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + logger := ctx.GetLogger() + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Compiling library \"{0}\"", library.Name) } libraryBuildPath := filepath.Join(buildPath, library.Name) @@ -144,12 +142,12 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr } if library.Layout == types.LIBRARY_RECURSIVE { - objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFilesRecursive(ctx, objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.DotALinkage { - archiveFile, err := builder_utils.ArchiveCompiledFiles(libraryBuildPath, library.Name+".a", objectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, library.Name+".a", objectFiles, buildProperties) if err != nil { return nil, i18n.WrapError(err) } @@ -159,14 +157,14 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr if library.UtilityFolder != "" { includes = append(includes, utils.WrapWithHyphenI(library.UtilityFolder)) } - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.UtilityFolder != "" { utilityBuildPath := filepath.Join(libraryBuildPath, constants.LIBRARY_FOLDER_UTILITY) - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } diff --git a/phases/linker.go b/phases/linker.go index d35f4993..9b3eb5c2 100644 --- a/phases/linker.go +++ b/phases/linker.go @@ -61,11 +61,8 @@ func (s *Linker) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err = link(objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, verbose, warningsLevel, logger) + err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,7 @@ func (s *Linker) Run(ctx *types.Context) error { return nil } -func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func link(ctx *types.Context, objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map) error { optRelax := addRelaxTrickIfATMEGA2560(buildProperties) objectFiles = utils.Map(objectFiles, wrapWithDoubleQuotes) @@ -81,12 +78,12 @@ func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath stri properties := buildProperties.Clone() properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] + optRelax - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = coreDotARelPath properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList - _, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, verbose, verbose, logger) + _, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, ctx.Verbose, ctx.Verbose, ctx.GetLogger()) return err } diff --git a/phases/sizer.go b/phases/sizer.go index b1bedf75..94d771dc 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -52,11 +52,8 @@ func (s *Sizer) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err := checkSize(buildProperties, verbose, warningsLevel, logger) + err := checkSize(ctx, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -64,10 +61,11 @@ func (s *Sizer) Run(ctx *types.Context) error { return nil } -func checkSize(buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func checkSize(ctx *types.Context, buildProperties properties.Map) error { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] maxTextSizeString := properties[constants.PROPERTY_UPLOAD_MAX_SIZE] maxDataSizeString := properties[constants.PROPERTY_UPLOAD_MAX_DATA_SIZE] @@ -89,7 +87,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin } } - textSize, dataSize, _, err := execSizeRecipe(properties, logger) + textSize, dataSize, _, err := execSizeRecipe(ctx, properties) if err != nil { logger.Println(constants.LOG_LEVEL_WARN, constants.MSG_SIZER_ERROR_NO_RULE) return nil @@ -127,8 +125,8 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin return nil } -func execSizeRecipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) { - out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, logger) +func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { + out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, ctx.GetLogger()) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) return diff --git a/phases/sketch_builder.go b/phases/sketch_builder.go index 04c7ded7..8a3158d9 100644 --- a/phases/sketch_builder.go +++ b/phases/sketch_builder.go @@ -46,9 +46,6 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { buildProperties := ctx.BuildProperties includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(sketchBuildPath) if err != nil { @@ -56,7 +53,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { } var objectFiles []string - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -64,7 +61,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC) if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() { - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } From 59d32510295ef1257bb9d64355d0eb24335915c7 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 13:39:01 +0200 Subject: [PATCH 41/68] Show the sizer commandline in verbose mode For some reason the sizer commandline was never shown. For consistency, it is now shown in verbose mode, just like the other commands. Signed-off-by: Matthijs Kooijman --- phases/sizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phases/sizer.go b/phases/sizer.go index 94d771dc..90d4e705 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -126,7 +126,7 @@ func checkSize(ctx *types.Context, buildProperties properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, ctx.GetLogger()) + out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, ctx.Verbose, false, ctx.GetLogger()) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) return From 79afcb609fb687d36020c2d2842adec2f580206c Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 13:47:47 +0200 Subject: [PATCH 42/68] Show stdout of preproc commands in verbose mode This also happens with the normal compilation commands, so why not with these? Normally these commands should not output to stdout, so this doesn't make any difference, but it makes things more consistent. Signed-off-by: Matthijs Kooijman --- gcc_preproc_runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 0f95917f..8156e06e 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -53,7 +53,7 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath verbose := ctx.Verbose logger := ctx.GetLogger() - _, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + _, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) if err != nil { return i18n.WrapError(err) } @@ -75,7 +75,7 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) } - stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) if err != nil { return "", i18n.WrapError(err) } From deaed666200625e6cb85bb9942e17cc84b026d76 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 14:56:55 +0200 Subject: [PATCH 43/68] Do not ignore command errors in ExecRecipeCollectStdErr Previously, this function would ignore any errors returned by `Run()` since the command is expected to fail in most cases. However, in addition to ignoring a non-zero exit code from the command, it would also ignore errors in running the command itself. With this commit, `ExecRecipeCollectStdErr()` simply returns all errors, but its caller checks the type of the error. If it is `ExitError`, this indicates a non-zero exit status, which is ignored. Otherwise, the error is reported as normal. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 4 ++-- container_find_includes.go | 8 +++++++- gcc_preproc_runner.go | 2 +- i18n/errors.go | 11 +++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 6e083283..5ebb7305 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -460,8 +460,8 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo buffer := &bytes.Buffer{} command.Stderr = buffer - command.Run() - return string(buffer.Bytes()), nil + err = command.Run() + return string(buffer.Bytes()), err } func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { diff --git a/container_find_includes.go b/container_find_includes.go index 91bafd7c..eb49d3ef 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -110,6 +110,7 @@ import ( "encoding/json" "io/ioutil" "os" + "os/exec" "path/filepath" "time" @@ -324,7 +325,12 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t } } else { stderr, err := GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) - if err != nil { + // Unwrap error and see if it is an ExitError. + // Ignore ExitErrors (e.g. gcc returning + // non-zero status), but bail out on other + // errors + _, is_exit_error := i18n.UnwrapError(err).(*exec.ExitError) + if err != nil && !is_exit_error { return i18n.WrapError(err) } include = IncludesFinderWithRegExp(ctx, stderr) diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 8156e06e..2e3dbe1c 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -77,7 +77,7 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) if err != nil { - return "", i18n.WrapError(err) + return string(stderr), i18n.WrapError(err) } return string(stderr), nil diff --git a/i18n/errors.go b/i18n/errors.go index ba2a86ea..b3617800 100644 --- a/i18n/errors.go +++ b/i18n/errors.go @@ -18,3 +18,14 @@ func WrapError(err error) error { } return errors.Wrap(err, 0) } + +func UnwrapError(err error) error { + // Perhaps go-errors can do this already in later versions? + // See https://github.com/go-errors/errors/issues/14 + switch e := err.(type) { + case *errors.Error: + return e.Err + default: + return err + } +} From 8563553cafb9f3cca5b602b03c62a5a61d55c8e6 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 15:25:15 +0200 Subject: [PATCH 44/68] Let ExecRecipeCollectStdErr return []byte for stderr Previously, the return value was converted to string. Letting callers convert to string makes it easier to write the sterr contents to os.Stderr again in a future commit. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 6 +++--- container_find_includes.go | 2 +- gcc_preproc_runner.go | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 5ebb7305..c3451759 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -452,16 +452,16 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo return command, nil } -func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (string, error) { +func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) { command, err := PrepareCommandForRecipe(buildProperties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) if err != nil { - return "", i18n.WrapError(err) + return nil, i18n.WrapError(err) } buffer := &bytes.Buffer{} command.Stderr = buffer err = command.Run() - return string(buffer.Bytes()), err + return buffer.Bytes(), err } func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { diff --git a/container_find_includes.go b/container_find_includes.go index eb49d3ef..912e3711 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -333,7 +333,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t if err != nil && !is_exit_error { return i18n.WrapError(err) } - include = IncludesFinderWithRegExp(ctx, stderr) + include = IncludesFinderWithRegExp(ctx, string(stderr)) } if include == "" { diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 2e3dbe1c..65fc91a5 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -61,10 +61,10 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath return nil } -func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (string, error) { +func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) ([]byte, error) { properties, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { - return "", i18n.WrapError(err) + return nil, i18n.WrapError(err) } verbose := ctx.Verbose @@ -77,10 +77,10 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) if err != nil { - return string(stderr), i18n.WrapError(err) + return stderr, i18n.WrapError(err) } - return string(stderr), nil + return stderr, nil } func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, error) { From fcc9c5d31d48b428ce4d6f41e019ce02d473e60c Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 15:46:04 +0200 Subject: [PATCH 45/68] Improve error handling in include detection For include detection, the preprocessor is run on all source files, collecting #included filenames from the stderr output, each of which are then resolved to a library to include. A caching mechanism is used to only run the preprocessor when needed. This commit improves the error handling during include detection in a number of ways: - When the preprocessor runs succesfully, processing stops for the current file. Previously, it would always look at stderr to find a missing include filename and only stop if none was found. - When the preprocessor fails, but no filename can be found, show the error preprocessor error. Previously, it would assume that the process was done and stop processing the file without any error. - When no library can be found for a missing include, show the stored error output instead of running the preprocessor again. Previously, the preprocessor would be run a second time, to (re)generate the error message. When the include filename comes from the cache and the preprocessor was not run yet, it is still run to show its errors to the user. This should be very rare, as normally changes that cause a cached filename to become unresolvable to a library also cause the file to be marked as changed, bypassing the cache. When this does happen, the preprocessor is now run using `GCCPreprocRunnerForDiscoveringIncludes()` instead of `GCCPreprocRunner()`, which ensures the preprocessor command is always exactly the same. Before this change, there could be specific circumstances where the first preprocessor run would generate an error, but where the second run would not show the error and include detection would continue as if nothing happened. One such circumstance is described in #230. Signed-off-by: Matthijs Kooijman --- container_find_includes.go | 44 +++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/container_find_includes.go b/container_find_includes.go index 912e3711..427e0177 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -119,6 +119,8 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" + + "github.com/go-errors/errors" ) type ContainerFindIncludes struct{} @@ -318,22 +320,33 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t if library, ok := sourceFile.Origin.(*types.Library); ok && library.UtilityFolder != "" { includes = append(includes, library.UtilityFolder) } + var preproc_err error + var preproc_stderr []byte if unchanged && cache.valid { include = cache.Next().Include if first && ctx.Verbose { ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_CACHED_INCLUDES, sourcePath) } } else { - stderr, err := GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) // Unwrap error and see if it is an ExitError. - // Ignore ExitErrors (e.g. gcc returning - // non-zero status), but bail out on other - // errors - _, is_exit_error := i18n.UnwrapError(err).(*exec.ExitError) - if err != nil && !is_exit_error { - return i18n.WrapError(err) + _, is_exit_error := i18n.UnwrapError(preproc_err).(*exec.ExitError) + if preproc_err == nil { + // Preprocessor successful, done + include = "" + } else if !is_exit_error || preproc_stderr == nil { + // Ignore ExitErrors (e.g. gcc returning + // non-zero status), but bail out on + // other errors + return i18n.WrapError(preproc_err) + } else { + include = IncludesFinderWithRegExp(ctx, string(preproc_stderr)) + if include == "" { + // No include found? Bail out. + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) + } } - include = IncludesFinderWithRegExp(ctx, string(stderr)) } if include == "" { @@ -345,8 +358,19 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - err := GCCPreprocRunner(ctx, sourcePath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, includes) - return i18n.WrapError(err) + if preproc_err == nil || preproc_stderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + if preproc_err == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New("Internal error in cache") + } + } + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) } // Add this library to the list of libraries, the From 8ec1f02bbbd26f95a0fc7c627c967bbf0421eccd Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 17:01:22 +0200 Subject: [PATCH 46/68] Merge ExecRecipeCollectStdErr into ExecRecipe This unifies these similar methods into a single method. The interface is additionally changed to: - Accepts a `Context` argument. - Allow for defaults and named arguments, using an `ExecOptions` struct that is passed as an argument. - Allow configuring command output handling in a flexible way. Instead of passing bools for some specific configurations, you can now pass either `Ignore`, `Show` or `Capture` for both stdout and stderr independently. By default, stdout is is shown when verbose is true, or ignored when verbose is false. Stderr is shown by default. - Actually redirect stdout to `/dev/null` when it is not needed (by leaving `command.Stdout` at nil). Previously, `ExecRecipe` would either show or capture stdout, and the captured output was usually just thrown away. To allow for even more reuse, the biggest part of `ExecRecipe` is extracted into a new `utils.ExecCommand()` function which executes an arbitrary `exec.Cmd` object, with configurable output redirection. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 53 ++++++++---------------------------------- gcc_preproc_runner.go | 9 ++----- phases/linker.go | 2 +- phases/sizer.go | 3 ++- recipe_runner.go | 12 +++++----- utils/utils.go | 41 ++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 58 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index c3451759..465a08bd 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -30,7 +30,6 @@ package builder_utils import ( - "bytes" "io" "os" "os/exec" @@ -213,7 +212,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, } if !objIsUpToDate { - _, err = ExecRecipe(properties, recipe, false, ctx.Verbose, ctx.Verbose, logger) + _, _, err = ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return "", i18n.WrapError(err) } @@ -380,7 +379,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile stri properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = archiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile - _, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.Verbose, ctx.Verbose, logger) + _, _, err := ExecRecipe(ctx, properties, constants.RECIPE_AR_PATTERN, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return "", i18n.WrapError(err) } @@ -389,40 +388,20 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile stri return archiveFilePath, nil } -func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) { - command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) +// See util.ExecCommand for stdout/stderr arguments +func ExecRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool, stdout int, stderr int) ([]byte, []byte, error) { + command, err := PrepareCommandForRecipe(ctx, buildProperties, recipe, removeUnsetProperties) if err != nil { - return nil, i18n.WrapError(err) - } - - if echoOutput { - printToStdOut := func(data []byte) { - logger.UnformattedWrite(os.Stdout, data) - } - stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}} - defer stdout.Flush() - command.Stdout = stdout - } - - printToStdErr := func(data []byte) { - logger.UnformattedWrite(os.Stderr, data) - } - stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}} - defer stderr.Flush() - command.Stderr = stderr - - if echoOutput { - err := command.Run() - return nil, i18n.WrapError(err) + return nil, nil, i18n.WrapError(err) } - bytes, err := command.Output() - return bytes, i18n.WrapError(err) + return utils.ExecCommand(ctx, command, stdout, stderr) } const COMMANDLINE_LIMIT = 30000 -func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) { +func PrepareCommandForRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool) (*exec.Cmd, error) { + logger := ctx.GetLogger() pattern := buildProperties[recipe] if pattern == constants.EMPTY_STRING { return nil, i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, recipe) @@ -445,25 +424,13 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo return nil, i18n.WrapError(err) } - if echoCommandLine { + if ctx.Verbose { logger.UnformattedFprintln(os.Stdout, commandLine) } return command, nil } -func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) { - command, err := PrepareCommandForRecipe(buildProperties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) - if err != nil { - return nil, i18n.WrapError(err) - } - - buffer := &bytes.Buffer{} - command.Stderr = buffer - err = command.Run() - return buffer.Bytes(), err -} - func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) } diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 65fc91a5..821a496a 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -51,9 +51,7 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) } - verbose := ctx.Verbose - logger := ctx.GetLogger() - _, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) + _, _, err = builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) if err != nil { return i18n.WrapError(err) } @@ -67,15 +65,12 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s return nil, i18n.WrapError(err) } - verbose := ctx.Verbose - logger := ctx.GetLogger() - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { //generate PREPROC_MACROS from RECIPE_CPP_PATTERN properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) } - stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, verbose, logger) + _, stderr, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Capture) if err != nil { return stderr, i18n.WrapError(err) } diff --git a/phases/linker.go b/phases/linker.go index 9b3eb5c2..2514b406 100644 --- a/phases/linker.go +++ b/phases/linker.go @@ -83,7 +83,7 @@ func link(ctx *types.Context, objectFiles []string, coreDotARelPath string, core properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList - _, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, ctx.Verbose, ctx.Verbose, ctx.GetLogger()) + _, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) return err } diff --git a/phases/sizer.go b/phases/sizer.go index 90d4e705..fa719e18 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -38,6 +38,7 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) @@ -126,7 +127,7 @@ func checkSize(ctx *types.Context, buildProperties properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, ctx.Verbose, false, ctx.GetLogger()) + out, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_SIZE_PATTERN, false /* stdout */, utils.Capture /* stderr */, utils.Show) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) return diff --git a/recipe_runner.go b/recipe_runner.go index bec23f90..bb1bb962 100644 --- a/recipe_runner.go +++ b/recipe_runner.go @@ -30,13 +30,15 @@ package builder import ( + "os" + "sort" + "strings" + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" - "os" - "sort" - "strings" + "github.com/arduino/arduino-builder/utils" ) type RecipeByPrefixSuffixRunner struct { @@ -51,8 +53,6 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties.Clone() - verbose := ctx.Verbose - recipes := findRecipes(buildProperties, s.Prefix, s.Suffix) properties := buildProperties.Clone() @@ -60,7 +60,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { if ctx.DebugLevel >= 10 { logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_RECIPE, recipe) } - _, err := builder_utils.ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, _, err := builder_utils.ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } diff --git a/utils/utils.go b/utils/utils.go index 3d26f971..cdb0069d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,6 +30,7 @@ package utils import ( + "bytes" "crypto/md5" "encoding/hex" "fmt" @@ -276,6 +277,46 @@ func PrepareCommand(pattern string, logger i18n.Logger, relativePath string) (*e return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger, relativePath) } +const ( + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal + ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise + Capture = 3 // Capture into buffer +) + +func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { + if stdout == Capture { + buffer := &bytes.Buffer{} + command.Stdout = buffer + } else if stdout == Show || stdout == ShowIfVerbose && ctx.Verbose { + command.Stdout = os.Stdout + } + + if stderr == Capture { + buffer := &bytes.Buffer{} + command.Stderr = buffer + } else if stderr == Show || stderr == ShowIfVerbose && ctx.Verbose { + command.Stderr = os.Stderr + } + + err := command.Start() + if err != nil { + return nil, nil, i18n.WrapError(err) + } + + err = command.Wait() + + var outbytes, errbytes []byte + if buf, ok := command.Stdout.(*bytes.Buffer); ok { + outbytes = buf.Bytes() + } + if buf, ok := command.Stderr.(*bytes.Buffer); ok { + errbytes = buf.Bytes() + } + + return outbytes, errbytes, i18n.WrapError(err) +} + func MapHas(aMap map[string]interface{}, key string) bool { _, ok := aMap[key] return ok From 4f014d170c9c4d4ae798b50b9982144260c70414 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 17:50:13 +0200 Subject: [PATCH 47/68] Merge some duplicate code into prepareGCCPreprocRecipeProperties Signed-off-by: Matthijs Kooijman --- gcc_preproc_runner.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 821a496a..5ba285fe 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -46,11 +46,6 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath return i18n.WrapError(err) } - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - _, _, err = builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) if err != nil { return i18n.WrapError(err) @@ -65,11 +60,6 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s return nil, i18n.WrapError(err) } - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - _, stderr, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Capture) if err != nil { return stderr, i18n.WrapError(err) @@ -87,6 +77,11 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties) + if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { + //generate PREPROC_MACROS from RECIPE_CPP_PATTERN + properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) + } + return properties, nil } From 5549e8a6bd65649f71d2011738ad868366b6cb2f Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 20:10:31 +0200 Subject: [PATCH 48/68] Let utils.ExecCommand print the command in verbose mode Previously, the command was printed by PrepareCommandForRecipe. Letting ExecCommand print seems more accurate, since it is only printed when it is actually run (though this already happened in practice). Additionally, the command can now be modified between PrepareCommandForRecipe and ExecCommand while preserving correct output. Since ExecCommand deals with a slice of arguments instead of a single command string, this requires merging them together into a proper commandline. Some care is taken to quote arguments containing spaces, quotes or backslashes, though this is mostly intended for display purposes. Arguments are only quoted when needed, regardless of whether they were quoted in the original pattern. Signed-off-by: Matthijs Kooijman --- test/utils_test.go | 19 +++++++++++++++++++ utils/utils.go | 32 +++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/test/utils_test.go b/test/utils_test.go index da8395fa..6ada0c29 100644 --- a/test/utils_test.go +++ b/test/utils_test.go @@ -70,6 +70,25 @@ func TestCommandLineParser(t *testing.T) { require.Equal(t, "/tmp/sketch321469072.cpp", parts[22]) } +func TestPrintableCommand(t *testing.T) { + parts := []string{ + "/path/to/dir with spaces/cmd", + "arg1", + "arg-\"with\"-quotes", + "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", + "arg with spaces", + "arg\twith\t\ttabs", + "lastarg", + } + correct := "\"/path/to/dir with spaces/cmd\"" + + " arg1 \"arg-\\\"with\\\"-quotes\"" + + " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + + " lastarg" + result := utils.PrintableCommand(parts) + require.Equal(t, correct, result) +} + func TestCommandLineParserError(t *testing.T) { command := "\"command missing quote" diff --git a/utils/utils.go b/utils/utils.go index cdb0069d..d29d1905 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -277,14 +277,36 @@ func PrepareCommand(pattern string, logger i18n.Logger, relativePath string) (*e return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger, relativePath) } +func printableArgument(arg string) string { + if strings.ContainsAny(arg, "\"\\ \t") { + arg = strings.Replace(arg, "\\", "\\\\", -1) + arg = strings.Replace(arg, "\"", "\\\"", -1) + return "\"" + arg + "\"" + } else { + return arg + } +} + +// Convert a command and argument slice back to a printable string. +// This adds basic escaping which is sufficient for debug output, but +// probably not for shell interpretation. This essentially reverses +// ParseCommandLine. +func PrintableCommand(parts []string) string { + return strings.Join(Map(parts, printableArgument), " ") +} + const ( - Ignore = 0 // Redirect to null - Show = 1 // Show on stdout/stderr as normal + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise - Capture = 3 // Capture into buffer + Capture = 3 // Capture into buffer ) func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { + if ctx.Verbose { + fmt.Println(PrintableCommand(command.Args)) + } + if stdout == Capture { buffer := &bytes.Buffer{} command.Stdout = buffer @@ -308,10 +330,10 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) var outbytes, errbytes []byte if buf, ok := command.Stdout.(*bytes.Buffer); ok { - outbytes = buf.Bytes() + outbytes = buf.Bytes() } if buf, ok := command.Stderr.(*bytes.Buffer); ok { - errbytes = buf.Bytes() + errbytes = buf.Bytes() } return outbytes, errbytes, i18n.WrapError(err) From 417b6af6deb6b6830b785072d4feb96fb76c1cd1 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Fri, 16 Jun 2017 20:18:16 +0200 Subject: [PATCH 49/68] Fix removal of -MMD option when running the preprocessor Usually, the `preproc.macros` recipe includes the C/C++ flags, and through that the `-MMD` flag to generate dependency files. However, since include detection passed an output file of `/dev/null` (or the equivalent on other operating systems), this causes gcc to try and generate a `/dev/null.d` file and fail. To prevent this, the `-MMD` flag was filtered out, but this filtering was applied to the `compiler.cpp.flags` variable, where it *usually* comes from. However, this is not necessarily true for all platforms. For example, the PIC32 platform used to have this flag in the `compiler.c.flags` variable and have `compiler.cpp.flags` include that. This prevented the flag from being filtered away and caused a failure. Due to previous changes, it is now possible for this filtering to happen after all variables have been replaced and the command to run was generated, but before actually running it. An extra advantage is that the filtering is more robust than the previous substring-based filtering. This fixes #230. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 4 ---- gcc_preproc_runner.go | 24 ++++++++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 465a08bd..b35d2998 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -431,10 +431,6 @@ func PrepareCommandForRecipe(ctx *types.Context, buildProperties properties.Map, return command, nil } -func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { - buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) -} - // CopyFile copies the contents of the file named src to the file named // by dst. The file will be created if it does not already exist. If the // destination file exists, all it's contents will be replaced by the contents diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index 5ba285fe..3c801f89 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -30,6 +30,7 @@ package builder import ( + "os/exec" "strings" "github.com/arduino/arduino-builder/builder_utils" @@ -37,16 +38,15 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "github.com/arduino/go-properties-map" ) func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error { - properties, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return i18n.WrapError(err) } - _, _, err = builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) + _, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } @@ -55,12 +55,12 @@ func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath } func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) ([]byte, error) { - properties, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return nil, i18n.WrapError(err) } - _, stderr, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Capture) + _, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture) if err != nil { return stderr, i18n.WrapError(err) } @@ -68,21 +68,29 @@ func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath s return stderr, nil } -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, error) { +func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (*exec.Cmd, error) { properties := ctx.BuildProperties.Clone() properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath includes = utils.Map(includes, utils.WrapWithHyphenI) properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) - builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties) if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { //generate PREPROC_MACROS from RECIPE_CPP_PATTERN properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) } - return properties, nil + cmd, err := builder_utils.PrepareCommandForRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true) + if err != nil { + return nil, i18n.WrapError(err) + } + + // Remove -MMD argument if present. Leaving it will make gcc try + // to create a /dev/null.d dependency file, which won't work. + cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" }) + + return cmd, nil } func GeneratePreprocPatternFromCompile(compilePattern string) string { From 36b60f5bf9a771c790bf5d6b149a5767dbf16006 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 30 Nov 2017 15:23:04 +0100 Subject: [PATCH 50/68] Pass Context to ObjFileIsUpToDate This does not use the passed variable yet, but prepares for a future commit. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 4 ++-- container_find_includes.go | 2 +- test/builder_utils_test.go | 36 ++++++++++++++++++++++++++---------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index b35d2998..73e8a04f 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -206,7 +206,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, return "", i18n.WrapError(err) } - objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) + objIsUpToDate, err := ObjFileIsUpToDate(ctx, properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) if err != nil { return "", i18n.WrapError(err) } @@ -223,7 +223,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) { +func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFile string) (bool, error) { sourceFile = filepath.Clean(sourceFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) diff --git a/container_find_includes.go b/container_find_includes.go index 427e0177..15cdc3b9 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -306,7 +306,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) + unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) if err != nil { return i18n.WrapError(err) } diff --git a/test/builder_utils_test.go b/test/builder_utils_test.go index c6fc941c..ef07ddf0 100644 --- a/test/builder_utils_test.go +++ b/test/builder_utils_test.go @@ -30,13 +30,15 @@ package test import ( - "github.com/arduino/arduino-builder/builder_utils" - "github.com/arduino/arduino-builder/utils" - "github.com/stretchr/testify/require" "io/ioutil" "os" "testing" "time" + + "github.com/arduino/arduino-builder/builder_utils" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/stretchr/testify/require" ) func sleep(t *testing.T) { @@ -52,27 +54,33 @@ func tempFile(t *testing.T, prefix string) string { } func TestObjFileIsUpToDateObjMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, "", "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, "", "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjOlder(t *testing.T) { + ctx := &types.Context{} + objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) depFile := tempFile(t, "dep") @@ -83,12 +91,14 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -99,12 +109,14 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer os.RemoveAll(depFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -122,12 +134,14 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -143,12 +157,14 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -166,7 +182,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { utils.WriteFile(depFile, sourceFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } From fc9a824a803b7f6c28f0e5b2510394bfff0582ef Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 13:53:54 +0100 Subject: [PATCH 51/68] Let ObjFileIsUpToDate output verbose debug output If -debug-level=20 is passed, whenever the cached file is not usable for whatever reason, a message is displayed. This should help debug caching problems. The messages are hardcoded in the source and not put into `constants`, since they are only debug messages. Signed-off-by: Matthijs Kooijman --- builder_utils/utils.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 73e8a04f..269357fa 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -227,6 +227,12 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil sourceFile = filepath.Clean(sourceFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) + logger := ctx.GetLogger() + debugLevel := ctx.DebugLevel + + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) + } sourceFileStat, err := os.Stat(sourceFile) if err != nil { @@ -236,6 +242,9 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil objectFileStat, err := os.Stat(objectFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -245,6 +254,9 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil dependencyFileStat, err := os.Stat(dependencyFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -252,9 +264,15 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile) + } return false, nil } if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile) + } return false, nil } @@ -274,10 +292,16 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil firstRow := rows[0] if !strings.HasSuffix(firstRow, ":") { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile") + } return false, nil } objFileInDepFile := firstRow[:len(firstRow)-1] if objFileInDepFile != objectFile { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile) + } return false, nil } @@ -287,12 +311,22 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil if err != nil && !os.IsNotExist(err) { // There is probably a parsing error of the dep file // Ignore the error and trigger a full rebuild anyway + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error()) + } return false, nil } if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row) + } return false, nil } if depStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile) + } return false, nil } } From cfc4fd00d35e823a9785583bec9fc6ab04a1247b Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 16:24:02 +0100 Subject: [PATCH 52/68] ContainerFindIncludes: Add some temporary variables This slightly cleans up a function call. Signed-off-by: Matthijs Kooijman --- container_find_includes.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/container_find_includes.go b/container_find_includes.go index 15cdc3b9..df0fca81 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -292,6 +292,8 @@ func writeCache(cache *includeCache, path string) error { func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { sourcePath := sourceFile.SourcePath(ctx) + depPath := sourceFile.DepfilePath(ctx) + objPath := sourceFile.ObjectPath(ctx) targetFilePath := utils.NULLFile() // TODO: This should perhaps also compare against the @@ -306,7 +308,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) + unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, objPath, depPath) if err != nil { return i18n.WrapError(err) } From 70f45ccff6b9b25433116f361bebdceea7b5b428 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 10 Jan 2018 15:16:03 +0100 Subject: [PATCH 53/68] Fix merge conflicts --- create_cmake_rule.go | 2 +- preprocess_sketch.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/create_cmake_rule.go b/create_cmake_rule.go index 5166447b..7038dd54 100644 --- a/create_cmake_rule.go +++ b/create_cmake_rule.go @@ -206,7 +206,7 @@ func canExportCmakeProject(ctx *types.Context) bool { } func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, linkerflags, linkDirectories *[]string, logger i18n.Logger) { - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, receipe, true, false, false, logger) + command, _ := builder_utils.PrepareCommandForRecipe(ctx, ctx.BuildProperties, receipe, true) for _, arg := range command.Args { if strings.HasPrefix(arg, "-D") { diff --git a/preprocess_sketch.go b/preprocess_sketch.go index a5f25691..04723681 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -48,7 +48,6 @@ type PreprocessSketch struct{} func (s *PreprocessSketch) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") commands := []types.Command{ - &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, &ArduinoPreprocessorRunner{}, } @@ -58,6 +57,8 @@ func (s *PreprocessSketch) Run(ctx *types.Context) error { commands = append(commands, &SketchSaver{}) } + GCCPreprocRunner(ctx, sourceFile, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, ctx.IncludeFolders) + for _, command := range commands { PrintRingNameIfDebug(ctx, command) err := command.Run(ctx) @@ -73,7 +74,7 @@ type ArduinoPreprocessorRunner struct{} func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { buildProperties := ctx.BuildProperties - targetFilePath := ctx.FileToRead + targetFilePath := constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E logger := ctx.GetLogger() properties := buildProperties.Clone() From 2fc66f3685edc3eba43b7a7580c42293c7fb9133 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 13 Dec 2017 23:11:42 +0100 Subject: [PATCH 54/68] update arduino-preprocessor to 0.1.4 --- test/helper_tools_downloader.go | 12 ++++++------ test/tools_loader_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index 556f126c..dc68eafe 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -114,13 +114,13 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }, }, - Tool{Name: "arduino-preprocessor", Version: "0.1.3", + Tool{Name: "arduino-preprocessor", Version: "0.1.4", OsUrls: []OsUrl{ - OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-i686-w64-mingw32.tar.bz2"}, - OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-x86_64-apple-darwin11.tar.bz2"}, - OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.3/arduino-preprocessor-0.1.3-armhf-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-x86_64-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-i686-w64-mingw32.tar.bz2"}, + OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-x86_64-apple-darwin11.tar.bz2"}, + OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-armhf-pc-linux-gnu.tar.bz2"}, }, }, } diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index 48afa6dd..798a5bca 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -71,8 +71,8 @@ func TestLoadTools(t *testing.T) { idx := 0 require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.3", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.3"), tools[idx].Folder) + require.Equal(t, "0.1.4", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.4"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) From a32d158dcf5e5cbd38bba73dc931adf89a814ef4 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 23 Jan 2018 14:40:51 +0100 Subject: [PATCH 55/68] Make progress smoother By adding more steps to the progress report we can have a smoother progress indication. However, the result will be slightly more than 100% due to counting the same operation multi-step operation twice. Overall, the result looks better anyway --- builder.go | 25 ++++++------------------- builder_utils/utils.go | 16 ++++++++++++++++ container_setup.go | 4 ++++ types/context.go | 9 +++++++++ 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/builder.go b/builder.go index 5955577d..f31355d8 100644 --- a/builder.go +++ b/builder.go @@ -35,6 +35,7 @@ import ( "strconv" "time" + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/phases" @@ -183,36 +184,22 @@ func (s *ParseHardwareAndDumpBuildProperties) Run(ctx *types.Context) error { } func runCommands(ctx *types.Context, commands []types.Command, progressEnabled bool) error { - commandsLength := len(commands) - progressForEachCommand := float32(100) / float32(commandsLength) - progress := float32(0) + ctx.Progress.PrintEnabled = progressEnabled + ctx.Progress.Progress = 0 + for _, command := range commands { PrintRingNameIfDebug(ctx, command) - printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, progress) + ctx.Progress.Steps = 100.0 / float64(len(commands)) + builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx) err := command.Run(ctx) if err != nil { return i18n.WrapError(err) } - progress += progressForEachCommand } - - printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, 100) - return nil } -func printProgressIfProgressEnabledAndMachineLogger(progressEnabled bool, ctx *types.Context, progress float32) { - if !progressEnabled { - return - } - - log := ctx.GetLogger() - if log.Name() == "machine" { - log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(float64(progress), 'f', 2, 32)) - } -} - func PrintRingNameIfDebug(ctx *types.Context, command types.Command) { if ctx.DebugLevel >= 10 { ctx.GetLogger().Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_COMMAND, strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name()) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 269357fa..e75faed6 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -34,6 +34,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "sync" @@ -44,6 +45,19 @@ import ( "github.com/arduino/go-properties-map" ) +func PrintProgressIfProgressEnabledAndMachineLogger(ctx *types.Context) { + + if !ctx.Progress.PrintEnabled { + return + } + + log := ctx.GetLogger() + if log.Name() == "machine" { + log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(ctx.Progress.Progress, 'f', 2, 32)) + ctx.Progress.Progress += ctx.Progress.Steps + } +} + func CompileFilesRecursive(ctx *types.Context, objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { objectFiles, err := CompileFiles(ctx, objectFiles, sourcePath, false, buildPath, buildProperties, includes) if err != nil { @@ -153,12 +167,14 @@ func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath errorsChan := make(chan error) doneChan := make(chan struct{}) + ctx.Progress.Steps = ctx.Progress.Steps / float64(len(sources)) var wg sync.WaitGroup wg.Add(len(sources)) for _, source := range sources { go func(source string) { defer wg.Done() + PrintProgressIfProgressEnabledAndMachineLogger(ctx) objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) if err != nil { errorsChan <- err diff --git a/container_setup.go b/container_setup.go index 93ee6d73..57749aa2 100644 --- a/container_setup.go +++ b/container_setup.go @@ -30,6 +30,7 @@ package builder import ( + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" ) @@ -54,7 +55,10 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) &AddMissingBuildPropertiesFromParentPlatformTxtFiles{}, } + ctx.Progress.Steps = ctx.Progress.Steps / float64(len(commands)) + for _, command := range commands { + builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx) PrintRingNameIfDebug(ctx, command) err := command.Run(ctx) if err != nil { diff --git a/types/context.go b/types/context.go index cf3c026e..a30cfc00 100644 --- a/types/context.go +++ b/types/context.go @@ -7,6 +7,12 @@ import ( "github.com/arduino/go-properties-map" ) +type ProgressStruct struct { + PrintEnabled bool + Steps float64 + Progress float64 +} + // Context structure type Context struct { // Build options @@ -81,6 +87,9 @@ type Context struct { Verbose bool DebugPreprocessor bool + // Dry run, only create progress map + Progress ProgressStruct + // Contents of a custom build properties file (line by line) CustomBuildProperties []string From 99397e2c1565611e3a7dfc1c5485fbfceb44264a Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 23 Jan 2018 14:48:16 +0100 Subject: [PATCH 56/68] Fix Travis build if new dependencies are added --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8ec2e23e..910c086e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - go get github.com/arduino/go-timeutils script: + - go get github.com/arduino/arduino-builder/arduino-builder - go build -o $HOME/arduino-builder -v github.com/arduino/arduino-builder/arduino-builder - export TEST_PACKAGES=`go list github.com/arduino/arduino-builder/...` - RES=0; I=0; for PKG in $TEST_PACKAGES; do go test -v -timeout 30m -covermode=count -coverprofile=coverage.$I.out $PKG; ((RES=RES+$?)); ((I++)); done; ( exit $RES ) From 17e7af249eec1ad6c007a2f1050434661612a946 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 23 Jan 2018 16:48:59 +0100 Subject: [PATCH 57/68] Fix import in example grpc client --- client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index 11184387..29e2ce7c 100644 --- a/client/client.go +++ b/client/client.go @@ -26,7 +26,7 @@ import ( "io" "log" - pb "arduino.cc/builder/grpc/proto" + pb "github.com/arduino/arduino-builder/grpc/proto" "golang.org/x/net/context" "google.golang.org/grpc" ) From 243b313bb10b090efa27a93e0d5e3b1d11e5151c Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 23 Jan 2018 18:00:04 +0100 Subject: [PATCH 58/68] Print ExecCommand cmdline only once with proper escape --- utils/utils.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index d29d1905..b1002c65 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -303,9 +303,6 @@ const ( ) func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { - if ctx.Verbose { - fmt.Println(PrintableCommand(command.Args)) - } if stdout == Capture { buffer := &bytes.Buffer{} From 571b92246a04de4597b540e893d91c75e080b130 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 23 Jan 2018 18:00:32 +0100 Subject: [PATCH 59/68] Use absolute filepath for preprocessed cpp Due to the changes in PR#236, paths must be absolute --- preprocess_sketch.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 04723681..344f8b82 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -51,13 +51,18 @@ func (s *PreprocessSketch) Run(ctx *types.Context) error { &ArduinoPreprocessorRunner{}, } + err := utils.EnsureFolderExists(ctx.PreprocPath) + if err != nil { + return i18n.WrapError(err) + } + if ctx.CodeCompleteAt != "" { commands = append(commands, &OutputCodeCompletions{}) } else { commands = append(commands, &SketchSaver{}) } - GCCPreprocRunner(ctx, sourceFile, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, ctx.IncludeFolders) + GCCPreprocRunner(ctx, sourceFile, filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), ctx.IncludeFolders) for _, command := range commands { PrintRingNameIfDebug(ctx, command) @@ -74,7 +79,7 @@ type ArduinoPreprocessorRunner struct{} func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { buildProperties := ctx.BuildProperties - targetFilePath := constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E + targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E) logger := ctx.GetLogger() properties := buildProperties.Clone() From 849faa1f40336d51c9b16726213619e3d01cc19d Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 24 Jan 2018 15:01:28 +0100 Subject: [PATCH 60/68] Wipe build directory if any additional file has been added/removed from the previous run --- test/create_build_options_map_test.go | 1 + test/store_build_options_map_test.go | 1 + types/context.go | 10 ++++++++++ wipeout_build_path_if_build_options_changed.go | 5 +++++ 4 files changed, 17 insertions(+) diff --git a/test/create_build_options_map_test.go b/test/create_build_options_map_test.go index dce062e8..d3fde15e 100644 --- a/test/create_build_options_map_test.go +++ b/test/create_build_options_map_test.go @@ -54,6 +54,7 @@ func TestCreateBuildOptionsMap(t *testing.T) { NoError(t, err) require.Equal(t, "{\n"+ + " \"additionalFiles\": \"\",\n"+ " \"builtInLibrariesFolders\": \"\",\n"+ " \"customBuildProperties\": \"\",\n"+ " \"fqbn\": \"fqbn\",\n"+ diff --git a/test/store_build_options_map_test.go b/test/store_build_options_map_test.go index 31c20f1d..ebd593c2 100644 --- a/test/store_build_options_map_test.go +++ b/test/store_build_options_map_test.go @@ -74,6 +74,7 @@ func TestStoreBuildOptionsMap(t *testing.T) { NoError(t, err) require.Equal(t, "{\n"+ + " \"additionalFiles\": \"\",\n"+ " \"builtInLibrariesFolders\": \"built-in libraries\",\n"+ " \"customBuildProperties\": \"custom=prop\",\n"+ " \"fqbn\": \"fqbn\",\n"+ diff --git a/types/context.go b/types/context.go index a30cfc00..dd3db4bc 100644 --- a/types/context.go +++ b/types/context.go @@ -1,6 +1,7 @@ package types import ( + "path/filepath" "strings" "github.com/arduino/arduino-builder/i18n" @@ -103,6 +104,14 @@ type Context struct { func (ctx *Context) ExtractBuildOptions() properties.Map { opts := make(properties.Map) + var additionalFilesRelative []string + if ctx.Sketch != nil { + for _, sketch := range ctx.Sketch.AdditionalFiles { + absPath := filepath.Dir(ctx.SketchLocation) + relPath := strings.TrimPrefix(sketch.Name, absPath) + additionalFilesRelative = append(additionalFilesRelative, relPath) + } + } opts["hardwareFolders"] = strings.Join(ctx.HardwareFolders, ",") opts["toolsFolders"] = strings.Join(ctx.ToolsFolders, ",") opts["builtInLibrariesFolders"] = strings.Join(ctx.BuiltInLibrariesFolders, ",") @@ -111,6 +120,7 @@ func (ctx *Context) ExtractBuildOptions() properties.Map { opts["fqbn"] = ctx.FQBN opts["runtime.ide.version"] = ctx.ArduinoAPIVersion opts["customBuildProperties"] = strings.Join(ctx.CustomBuildProperties, ",") + opts["additionalFiles"] = strings.Join(additionalFilesRelative, ",") return opts } diff --git a/wipeout_build_path_if_build_options_changed.go b/wipeout_build_path_if_build_options_changed.go index 0764345d..141b2fca 100644 --- a/wipeout_build_path_if_build_options_changed.go +++ b/wipeout_build_path_if_build_options_changed.go @@ -33,6 +33,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" @@ -85,6 +86,10 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { if err != nil { return i18n.WrapError(err) } + // if build path is inside the sketch folder, also wipe ctx.AdditionalFiles + if strings.Contains(ctx.BuildPath, filepath.Dir(ctx.SketchLocation)) { + ctx.Sketch.AdditionalFiles = ctx.Sketch.AdditionalFiles[:0] + } for _, file := range files { os.RemoveAll(filepath.Join(buildPath, file.Name())) } From 9b0a81a95695592857c504fba238b3d48323cf46 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 24 Jan 2018 15:14:30 +0100 Subject: [PATCH 61/68] Wipe build dir only if txt build rules has changed All other source files are "tracked" by .d dependency files Solves https://github.com/arduino/arduino-builder/issues/251 --- builder_utils/utils.go | 26 +++++++++++++++++++ ...out_build_path_if_build_options_changed.go | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index e75faed6..870a6b94 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -392,6 +392,32 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) return true } +func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile string) bool { + + targetFileStat, err := os.Stat(targetFile) + if err == nil { + files, err := findAllFilesInFolder(corePath, true) + if err != nil { + return true + } + for _, file := range files { + // report changes only for .txt files + if filepath.Ext(file) != ".txt" { + continue + } + fileStat, err := os.Stat(file) + if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { + return true + } + } + if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) { + return TXTBuildRulesHaveChanged(targetCorePath, constants.EMPTY_STRING, targetFile) + } + return false + } + return true +} + func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map) (string, error) { logger := ctx.GetLogger() archiveFilePath := filepath.Join(buildPath, archiveFile) diff --git a/wipeout_build_path_if_build_options_changed.go b/wipeout_build_path_if_build_options_changed.go index 141b2fca..87048e77 100644 --- a/wipeout_build_path_if_build_options_changed.go +++ b/wipeout_build_path_if_build_options_changed.go @@ -73,7 +73,7 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] realCoreFolder := utils.GetParentFolder(coreFolder, 2) jsonPath := filepath.Join(ctx.BuildPath, constants.BUILD_OPTIONS_FILE) - coreHasChanged := builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, jsonPath) + coreHasChanged := builder_utils.TXTBuildRulesHaveChanged(realCoreFolder, targetCoreFolder, jsonPath) if opts.Equals(prevOpts) && !coreHasChanged { return nil From 9b7af5e23343926e90fe70af4e17a763892a2713 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 24 Jan 2018 15:23:04 +0100 Subject: [PATCH 62/68] update arduino-preprocessor to 0.1.5 --- test/helper_tools_downloader.go | 12 ++++++------ test/tools_loader_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index dc68eafe..db6dc9dc 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -114,13 +114,13 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, }, }, - Tool{Name: "arduino-preprocessor", Version: "0.1.4", + Tool{Name: "arduino-preprocessor", Version: "0.1.5", OsUrls: []OsUrl{ - OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-i686-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-x86_64-pc-linux-gnu.tar.bz2"}, - OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-i686-w64-mingw32.tar.bz2"}, - OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-x86_64-apple-darwin11.tar.bz2"}, - OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.4/arduino-preprocessor-0.1.4-armhf-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-x86_64-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-i686-w64-mingw32.tar.bz2"}, + OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-x86_64-apple-darwin11.tar.bz2"}, + OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-arm-linux-gnueabihf.tar.bz2"}, }, }, } diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index 798a5bca..926bca37 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -71,8 +71,8 @@ func TestLoadTools(t *testing.T) { idx := 0 require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.4", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.4"), tools[idx].Folder) + require.Equal(t, "0.1.5", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.5"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) @@ -151,8 +151,8 @@ func TestLoadLotsOfTools(t *testing.T) { require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder) idx++ require.Equal(t, "arduino-preprocessor", tools[idx].Name) - require.Equal(t, "0.1.1", tools[idx].Version) - require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.1"), tools[idx].Folder) + require.Equal(t, "0.1.5", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.5"), tools[idx].Folder) idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) From 748c70d06261dd74c87a2473f26de707e7f17827 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 24 Jan 2018 16:23:01 +0100 Subject: [PATCH 63/68] Revert "[CMAKE] include Arduino.h in main sketch file" This reverts commit e7600b54770fe353094090049dd87f6b9fa91032. --- sketch_source_merger.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sketch_source_merger.go b/sketch_source_merger.go index f2c91cb6..bbfaad0d 100644 --- a/sketch_source_merger.go +++ b/sketch_source_merger.go @@ -31,10 +31,9 @@ package builder import ( - "regexp" - "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" + "regexp" ) type SketchSourceMerger struct{} @@ -44,12 +43,12 @@ func (s *SketchSourceMerger) Run(ctx *types.Context) error { lineOffset := 0 includeSection := "" - includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n" - lineOffset++ if !sketchIncludesArduinoH(&sketch.MainFile) { includeSection += "#include \n" lineOffset++ } + includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n" + lineOffset++ ctx.IncludeSection = includeSection source := includeSection From eab9474c28238a0cdd0ce90a444633c6694b2eb4 Mon Sep 17 00:00:00 2001 From: Riccardo Rizzo Date: Wed, 14 Mar 2018 17:00:20 +0100 Subject: [PATCH 64/68] sort objectFiles slice alphabetically this makes the compilation deterministic. fixes https://github.com/arduino/Arduino/issues/7278 --- builder_utils/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 870a6b94..ec76ad3a 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -34,6 +34,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -200,6 +201,7 @@ func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath for objectFile := range objectFilesChan { objectFiles = append(objectFiles, objectFile) } + sort.Strings(objectFiles) return objectFiles, nil } } From ffe96696f3447613e7975e7ba99f63db4349ed4c Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 12 Apr 2018 09:49:32 +0200 Subject: [PATCH 65/68] Introduce compiler.libraries.ldflags Choosing compiler.ldflags for precompiled libraries was an unfortunate choice, since some cores define it in "wrong" places (or it's already in use for other flags). Deprecate the old behaviour and add a specific flag that should be added to recipe.combine in the right position to let the linker find the libraries. --- constants/constants.go | 1 + phases/libraries_builder.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/constants/constants.go b/constants/constants.go index cca3ceaf..53f87337 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -49,6 +49,7 @@ const BUILD_PROPERTIES_BUILD_VARIANT = "build.variant" const BUILD_PROPERTIES_BUILD_VARIANT_PATH = "build.variant.path" const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" +const BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS = "compiler.libraries.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" const BUILD_PROPERTIES_COMPILER_PATH = "compiler.path" const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index 9cc1fecc..34c63014 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -90,7 +90,7 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib name = strings.Replace(name, "lib", "", 1) libs_cmd += "-l" + name + " " } - ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LDFLAGS] += "\"-L" + path + "\" " + libs_cmd + ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS] = "\"-L" + path + "\" " + libs_cmd } } return nil From 2b1cfdd6e26fd9d9722693a14582346a800667c1 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 20 Apr 2018 09:36:35 +0200 Subject: [PATCH 66/68] Add runtime.ide.path env variable Partially fixes https://github.com/arduino/Arduino/issues/7160 --- constants/constants.go | 1 + setup_build_properties.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/constants/constants.go b/constants/constants.go index 53f87337..7254c2df 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -66,6 +66,7 @@ const BUILD_PROPERTIES_PATTERN = "pattern" const BUILD_PROPERTIES_PID = "pid" const BUILD_PROPERTIES_PREPROCESSED_FILE_PATH = "preprocessed_file_path" const BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH = "runtime.hardware.path" +const BUILD_PROPERTIES_RUNTIME_IDE_PATH = "runtime.ide.path" const BUILD_PROPERTIES_RUNTIME_OS = "runtime.os" const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" const BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX = "runtime.tools." diff --git a/setup_build_properties.go b/setup_build_properties.go index b18d643a..c1983551 100644 --- a/setup_build_properties.go +++ b/setup_build_properties.go @@ -30,6 +30,7 @@ package builder import ( + "os" "path/filepath" "strconv" "strings" @@ -64,12 +65,20 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error { } buildProperties[constants.BUILD_PROPERTIES_BUILD_ARCH] = strings.ToUpper(targetPlatform.PlatformId) + // get base folder and use it to populate BUILD_PROPERTIES_RUNTIME_IDE_PATH (arduino and arduino-builder live in the same dir) + ex, err := os.Executable() + exPath := "" + if err == nil { + exPath = filepath.Dir(ex) + } + buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE] = ctx.BuildCore buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] = filepath.Join(actualPlatform.Folder, constants.FOLDER_CORES, buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE]) buildProperties[constants.BUILD_PROPERTIES_BUILD_SYSTEM_PATH] = filepath.Join(actualPlatform.Folder, constants.FOLDER_SYSTEM) buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..") buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion + buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_PATH] = exPath buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName() From 55251c59d8d54edfc682445ec19ebba77d9f1f99 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 22 May 2018 11:54:41 +0200 Subject: [PATCH 67/68] Move arduino-proprocessor under experimental flag --- arduino-builder/main.go | 8 ++++++++ builder.go | 12 ++++++++++++ preprocess_sketch.go | 4 ++-- types/context.go | 3 +++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index ae4a92d1..89c31bdb 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -88,6 +88,7 @@ const FLAG_DAEMON = "daemon" const FLAG_VID_PID = "vid-pid" const FLAG_JOBS = "jobs" const FLAG_TRACE = "trace" +const FLAG_EXPERIMENTAL = "experimental" type foldersFlag []string @@ -149,6 +150,7 @@ var daemonFlag *bool var vidPidFlag *string var jobsFlag *int var traceFlag *bool +var experimentalFeatures *bool func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -176,6 +178,7 @@ func init() { vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") traceFlag = flag.Bool(FLAG_TRACE, false, "traces the whole process lifecycle") + experimentalFeatures = flag.Bool(FLAG_EXPERIMENTAL, false, "enables experimental features") } func main() { @@ -222,6 +225,11 @@ func main() { ctx := &types.Context{} + // place here all experimental features that should live under this flag + if *experimentalFeatures { + ctx.UseArduinoPreprocessor = true + } + if *daemonFlag { var loggerBuffer []string logger := i18n.AccumulatorLogger{} diff --git a/builder.go b/builder.go index f31355d8..6c143996 100644 --- a/builder.go +++ b/builder.go @@ -142,6 +142,18 @@ func (s *Builder) Run(ctx *types.Context) error { return otherErr } +type PreprocessSketch struct{} + +func (s *PreprocessSketch) Run(ctx *types.Context) error { + var commands []types.Command + if ctx.UseArduinoPreprocessor { + commands = append(commands, &PreprocessSketchArduino{}) + } else { + commands = append(commands, &ContainerAddPrototypes{}) + } + return runCommands(ctx, commands, true) +} + type Preprocess struct{} func (s *Preprocess) Run(ctx *types.Context) error { diff --git a/preprocess_sketch.go b/preprocess_sketch.go index 344f8b82..d36c04c5 100644 --- a/preprocess_sketch.go +++ b/preprocess_sketch.go @@ -43,9 +43,9 @@ import ( "github.com/arduino/arduino-builder/utils" ) -type PreprocessSketch struct{} +type PreprocessSketchArduino struct{} -func (s *PreprocessSketch) Run(ctx *types.Context) error { +func (s *PreprocessSketchArduino) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") commands := []types.Command{ &ArduinoPreprocessorRunner{}, diff --git a/types/context.go b/types/context.go index dd3db4bc..e9368c08 100644 --- a/types/context.go +++ b/types/context.go @@ -100,6 +100,9 @@ type Context struct { // Reuse old tools since the backing storage didn't change CanUseCachedTools bool + + // Experimental: use arduino-preprocessor to create prototypes + UseArduinoPreprocessor bool } func (ctx *Context) ExtractBuildOptions() properties.Map { From 14b777f14c247a0fa7e96a43683b14b14a62cf2e Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 30 May 2018 11:02:19 +0200 Subject: [PATCH 68/68] Allow multiple precompiled libraries to be included and linked --- phases/libraries_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index 34c63014..dc0fdf12 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -90,7 +90,7 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib name = strings.Replace(name, "lib", "", 1) libs_cmd += "-l" + name + " " } - ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS] = "\"-L" + path + "\" " + libs_cmd + ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS] += "\"-L" + path + "\" " + libs_cmd + " " } } return nil