Implemented json output in 'compile' command (#1065)

* Added Compile json output

* Factored out methods to convert Libraries into their rpc counterpart

* Give some more info in compiler json output

* Added simple smoke-test for compile --format json

* Output in json even in case of error

* Added binary size to compile json output

* lib list: always output rows in table output, even if the "library.Release" field is not set

Author: cmaglie

arduino/libraries/libraries.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
 	semver "go.bug.st/relaxed-semver"
@@ -85,6 +86,49 @@ func (library *Library) String() string {
 	return library.Name + "@" + library.Version.String()
 }
 
+// ToRPCLibrary converts this library into an rpc.Library
+func (library *Library) ToRPCLibrary() *rpc.Library {
+	pathOrEmpty := func(p *paths.Path) string {
+		if p == nil {
+			return ""
+		}
+		return p.String()
+	}
+	platformOrEmpty := func(p *cores.PlatformRelease) string {
+		if p == nil {
+			return ""
+		}
+		return p.String()
+	}
+	return &rpc.Library{
+		Name:              library.Name,
+		Author:            library.Author,
+		Maintainer:        library.Maintainer,
+		Sentence:          library.Sentence,
+		Paragraph:         library.Paragraph,
+		Website:           library.Website,
+		Category:          library.Category,
+		Architectures:     library.Architectures,
+		Types:             library.Types,
+		InstallDir:        pathOrEmpty(library.InstallDir),
+		SourceDir:         pathOrEmpty(library.SourceDir),
+		UtilityDir:        pathOrEmpty(library.UtilityDir),
+		Location:          library.Location.ToRPCLibraryLocation(),
+		ContainerPlatform: platformOrEmpty(library.ContainerPlatform),
+		Layout:            library.Layout.ToRPCLibraryLayout(),
+		RealName:          library.RealName,
+		DotALinkage:       library.DotALinkage,
+		Precompiled:       library.Precompiled,
+		LdFlags:           library.LDflags,
+		IsLegacy:          library.IsLegacy,
+		Version:           library.Version.String(),
+		License:           library.License,
+		Examples:          library.Examples.AsStrings(),
+		ProvidesIncludes:  library.DeclaredHeaders(),
+		CompatibleWith:    library.CompatibleWith,
+	}
+}
+
 // SupportsAnyArchitectureIn returns true if any of the following is true:
 // - the library supports at least one of the given architectures
 // - the library is architecture independent

arduino/libraries/librariesindex/index.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/arduino/resources"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	semver "go.bug.st/relaxed-semver"
 )
 
@@ -58,6 +59,21 @@ type Release struct {
 	Library *Library `json:"-"`
 }
 
+// ToRPCLibraryRelease transform this Release into a rpc.LibraryRelease
+func (r *Release) ToRPCLibraryRelease() *rpc.LibraryRelease {
+	return &rpc.LibraryRelease{
+		Author:        r.Author,
+		Version:       r.Version.String(),
+		Maintainer:    r.Maintainer,
+		Sentence:      r.Sentence,
+		Paragraph:     r.Paragraph,
+		Website:       r.Website,
+		Category:      r.Category,
+		Architectures: r.Architectures,
+		Types:         r.Types,
+	}
+}
+
 // GetName returns the name of this library.
 func (r *Release) GetName() string {
 	return r.Library.Name

cli/compile/compile.go

@@ -16,10 +16,12 @@
 package compile
 
 import (
+	"bytes"
 	"context"
 	"os"
 
 	"github.com/arduino/arduino-cli/cli/feedback"
+	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/configuration"
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
@@ -124,7 +126,7 @@ func run(cmd *cobra.Command, args []string) {
 	// the config file and the env vars.
 	exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")
 
-	_, err = compile.Compile(context.Background(), &rpc.CompileReq{
+	compileReq := &rpc.CompileReq{
 		Instance:         inst,
 		Fqbn:             fqbn,
 		SketchPath:       sketchPath.String(),
@@ -142,15 +144,19 @@ func run(cmd *cobra.Command, args []string) {
 		OptimizeForDebug: optimizeForDebug,
 		Clean:            clean,
 		ExportBinaries:   exportBinaries,
-	}, os.Stdout, os.Stderr, configuration.Settings.GetString("logging.level") == "debug")
-
-	if err != nil {
-		feedback.Errorf("Error during build: %v", err)
-		os.Exit(errorcodes.ErrGeneric)
+	}
+	compileOut := new(bytes.Buffer)
+	compileErr := new(bytes.Buffer)
+	verboseCompile := configuration.Settings.GetString("logging.level") == "debug"
+	var compileRes *rpc.CompileResp
+	if output.OutputFormat == "json" {
+		compileRes, err = compile.Compile(context.Background(), compileReq, compileOut, compileErr, verboseCompile)
+	} else {
+		compileRes, err = compile.Compile(context.Background(), compileReq, os.Stdout, os.Stderr, verboseCompile)
 	}
 
-	if uploadAfterCompile {
-		_, err := upload.Upload(context.Background(), &rpc.UploadReq{
+	if err == nil && uploadAfterCompile {
+		uploadReq := &rpc.UploadReq{
 			Instance:   inst,
 			Fqbn:       fqbn,
 			SketchPath: sketchPath.String(),
@@ -159,13 +165,32 @@ func run(cmd *cobra.Command, args []string) {
 			Verify:     verify,
 			ImportDir:  buildPath,
 			Programmer: programmer,
-		}, os.Stdout, os.Stderr)
-
+		}
+		var err error
+		if output.OutputFormat == "json" {
+			// TODO: do not print upload output in json mode
+			uploadOut := new(bytes.Buffer)
+			uploadErr := new(bytes.Buffer)
+			_, err = upload.Upload(context.Background(), uploadReq, uploadOut, uploadErr)
+		} else {
+			_, err = upload.Upload(context.Background(), uploadReq, os.Stdout, os.Stderr)
+		}
 		if err != nil {
 			feedback.Errorf("Error during Upload: %v", err)
 			os.Exit(errorcodes.ErrGeneric)
 		}
 	}
+
+	feedback.PrintResult(&compileResult{
+		CompileOut:    compileOut.String(),
+		CompileErr:    compileErr.String(),
+		BuilderResult: compileRes,
+		Success:       err == nil,
+	})
+	if err != nil && output.OutputFormat != "json" {
+		feedback.Errorf("Error during build: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
 }
 
 // initSketchPath returns the current working directory
@@ -182,3 +207,19 @@ func initSketchPath(sketchPath *paths.Path) *paths.Path {
 	logrus.Infof("Reading sketch from dir: %s", wd)
 	return wd
 }
+
+type compileResult struct {
+	CompileOut    string           `json:"compiler_out"`
+	CompileErr    string           `json:"compiler_err"`
+	BuilderResult *rpc.CompileResp `json:"builder_result"`
+	Success       bool             `json:"success"`
+}
+
+func (r *compileResult) Data() interface{} {
+	return r
+}
+
+func (r *compileResult) String() string {
+	// The output is already printed via os.Stdout/os.Stdin
+	return ""
+}

cli/lib/list.go

@@ -140,19 +140,22 @@ func (ir installedResult) String() string {
 			location = lib.GetContainerPlatform()
 		}
 
+		available := ""
+		sentence := ""
 		if libMeta.GetRelease() != nil {
-			available := libMeta.GetRelease().GetVersion()
-			if available == "" {
-				available = "-"
-			}
-			sentence := lib.Sentence
-			if sentence == "" {
-				sentence = "-"
-			} else if len(sentence) > 40 {
-				sentence = sentence[:37] + "..."
-			}
-			t.AddRow(name, lib.Version, available, location, sentence)
+			available = libMeta.GetRelease().GetVersion()
+			sentence = lib.Sentence
+		}
+
+		if available == "" {
+			available = "-"
+		}
+		if sentence == "" {
+			sentence = "-"
+		} else if len(sentence) > 40 {
+			sentence = sentence[:37] + "..."
 		}
+		t.AddRow(name, lib.Version, available, location, sentence)
 	}
 
 	return t.Render()

commands/compile/compile.go

@@ -189,6 +189,21 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	builderCtx.SetLogger(i18n.LoggerToCustomStreams{Stdout: outStream, Stderr: errStream})
 	builderCtx.Clean = req.GetClean()
 
+	// Use defer() to create an rpc.CompileResp with all the information available at the
+	// moment of return.
+	defer func() {
+		if r != nil {
+			importedLibs := []*rpc.Library{}
+			for _, lib := range builderCtx.ImportedLibraries {
+				importedLibs = append(importedLibs, lib.ToRPCLibrary())
+			}
+
+			r.BuildPath = builderCtx.BuildPath.String()
+			r.UsedLibraries = importedLibs
+			r.ExecutableSectionsSize = builderCtx.ExecutableSectionsSize.ToRPCExecutableSectionSizeArray()
+		}
+	}()
+
 	// if --preprocess or --show-properties were passed, we can stop here
 	if req.GetShowProperties() {
 		return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx)
@@ -198,7 +213,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 
 	// if it's a regular build, go on...
 	if err := builder.RunBuilder(builderCtx); err != nil {
-		return nil, err
+		return &rpc.CompileResp{}, err
 	}
 
 	// If the export directory is set we assume you want to export the binaries

commands/lib/list.go

@@ -99,13 +99,12 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListReq) (*rpc.LibraryList
 		if nameFilter != "" && strings.ToLower(lib.Library.Name) != nameFilter {
 			continue
 		}
-		libtmp, err := GetOutputLibrary(lib.Library)
-		if err != nil {
-			return nil, err
+		var release *rpc.LibraryRelease
+		if lib.Available != nil {
+			release = lib.Available.ToRPCLibraryRelease()
 		}
-		release := GetOutputRelease(lib.Available)
 		instaledLib = append(instaledLib, &rpc.InstalledLibrary{
-			Library: libtmp,
+			Library: lib.Library.ToRPCLibrary(),
 			Release: release,
 		})
 	}
@@ -136,69 +135,3 @@ func listLibraries(lm *librariesmanager.LibrariesManager, updatable bool, all bo
 	}
 	return res
 }
-
-// GetOutputLibrary FIXMEDOC
-func GetOutputLibrary(lib *libraries.Library) (*rpc.Library, error) {
-	insdir := ""
-	if lib.InstallDir != nil {
-		insdir = lib.InstallDir.String()
-	}
-	srcdir := ""
-	if lib.SourceDir != nil {
-		srcdir = lib.SourceDir.String()
-	}
-	utldir := ""
-	if lib.UtilityDir != nil {
-		utldir = lib.UtilityDir.String()
-	}
-	cntplat := ""
-	if lib.ContainerPlatform != nil {
-		cntplat = lib.ContainerPlatform.String()
-	}
-
-	return &rpc.Library{
-		Name:              lib.Name,
-		Author:            lib.Author,
-		Maintainer:        lib.Maintainer,
-		Sentence:          lib.Sentence,
-		Paragraph:         lib.Paragraph,
-		Website:           lib.Website,
-		Category:          lib.Category,
-		Architectures:     lib.Architectures,
-		Types:             lib.Types,
-		InstallDir:        insdir,
-		SourceDir:         srcdir,
-		UtilityDir:        utldir,
-		Location:          lib.Location.ToRPCLibraryLocation(),
-		ContainerPlatform: cntplat,
-		Layout:            lib.Layout.ToRPCLibraryLayout(),
-		RealName:          lib.RealName,
-		DotALinkage:       lib.DotALinkage,
-		Precompiled:       lib.Precompiled,
-		LdFlags:           lib.LDflags,
-		IsLegacy:          lib.IsLegacy,
-		Version:           lib.Version.String(),
-		License:           lib.License,
-		Examples:          lib.Examples.AsStrings(),
-		ProvidesIncludes:  lib.DeclaredHeaders(),
-		CompatibleWith:    lib.CompatibleWith,
-	}, nil
-}
-
-// GetOutputRelease FIXMEDOC
-func GetOutputRelease(lib *librariesindex.Release) *rpc.LibraryRelease { //
-	if lib != nil {
-		return &rpc.LibraryRelease{
-			Author:        lib.Author,
-			Version:       lib.Version.String(),
-			Maintainer:    lib.Maintainer,
-			Sentence:      lib.Sentence,
-			Paragraph:     lib.Paragraph,
-			Website:       lib.Website,
-			Category:      lib.Category,
-			Architectures: lib.Architectures,
-			Types:         lib.Types,
-		}
-	}
-	return &rpc.LibraryRelease{}
-}

legacy/builder/phases/sizer.go

@@ -88,6 +88,21 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error {
 		}
 	}
 
+	ctx.ExecutableSectionsSize = []types.ExecutableSectionSize{
+		{
+			Name:    "text",
+			Size:    textSize,
+			MaxSize: maxTextSize,
+		},
+	}
+	if maxDataSize > 0 {
+		ctx.ExecutableSectionsSize = append(ctx.ExecutableSectionsSize, types.ExecutableSectionSize{
+			Name:    "data",
+			Size:    dataSize,
+			MaxSize: maxDataSize,
+		})
+	}
+
 	if textSize > maxTextSize {
 		logger.Println(constants.LOG_LEVEL_ERROR, constants.MSG_SIZER_TEXT_TOO_BIG)
 		return errors.New("text section exceeds available space in board")