[breaking] Updated gRPC Platform API (#2357)

* Updated gRPC Pltform API, regenerated API

* Adapted cores.Platform and PlatformRelease to the new gRPC API

* Fixed search_test.go

* Removed gRPC PlatformList command

* Other adaptation of platform structures

* Adapt arguuments completion to use PlatformSearch instead of PlatformList

* Adapt 'core list' to use PlatformSearch instead of PlatformList

* Adapt 'core upgrade' command to use PlatformSearch instead of PlatformList

* Adapted some integration tests

* Fix integreation test

* apply changes to search vidpid

* Better handling of 'core list' results

* Better handling of 'core search' results

* Better handling of 'core outdated' results

* add 'orderedmap' data structure

* Made orderedmap more generic

* fix regression on 'ParseReference'

* wip: fix 'core' integrationtests

* wip: fix 'core' sorting tests

* fix regression which skipped mannually instaled core in core list

* wip: add more 'core' integration tests

* regression: all flag takes precedence above updatable in core list

* lint: ignore unexported-return (revive)

* license: regenerate and add missin headers

* tests: fix 'board' integrations

* fix: regression not showing manually installed platform in 'core list'

* wip: add test to orderedmap

* add more orderdmap tests

* orderdmap: add json tests

* update DOCS

* apply CR suggestions

* fix proto numeration

* docs: update to release 0.36.0

---------

Co-authored-by: Alessio Perugini <[email protected]>

Author: cmaglie

Diff for: .golangci.yml

@@ -62,7 +62,6 @@ linters-settings:
       - name: redefines-builtin-id
       - name: superfluous-else
       - name: time-naming
-      - name: unexported-return
       - name: unreachable-code
       - name: var-declaration
       - name: defer

Diff for: arduino/cores/cores.go

@@ -37,14 +37,13 @@ import (
 
 // Platform represents a platform package.
 type Platform struct {
-	Architecture      string // The name of the architecture of this package.
-	Name              string
-	Category          string
+	Architecture      string                                       // The name of the architecture of this package.
 	Releases          map[semver.NormalizedString]*PlatformRelease // The Releases of this platform, labeled by version.
 	Package           *Package                                     `json:"-"`
 	ManuallyInstalled bool                                         // true if the Platform has been installed without the CLI
-	Deprecated        bool                                         // true if the Platform has been deprecated
+	Deprecated        bool                                         // true if the latest PlatformRelease of this Platform has been deprecated
 	Indexed           bool                                         // true if the Platform has been indexed from additional-urls
+	Latest            *semver.Version                              `json:"-"`
 }
 
 // PlatformReleaseHelp represents the help URL for this Platform release
@@ -54,12 +53,15 @@ type PlatformReleaseHelp struct {
 
 // PlatformRelease represents a release of a plaform package.
 type PlatformRelease struct {
+	Name                    string
+	Category                string
 	Resource                *resources.DownloadResource
 	Version                 *semver.Version
 	BoardsManifest          []*BoardManifest
 	ToolDependencies        ToolDependencies
 	DiscoveryDependencies   DiscoveryDependencies
 	MonitorDependencies     MonitorDependencies
+	Deprecated              bool
 	Help                    PlatformReleaseHelp           `json:"-"`
 	Platform                *Platform                     `json:"-"`
 	Properties              *properties.Map               `json:"-"`
@@ -407,7 +409,7 @@ func (release *PlatformRelease) MarshalJSON() ([]byte, error) {
 		ID:        release.Platform.String(),
 		Installed: release.Version.String(),
 		Latest:    latestStr,
-		Name:      release.Platform.Name,
+		Name:      release.Name,
 	})
 }
 

Diff for: arduino/cores/packageindex/index.go

@@ -222,11 +222,11 @@ func IndexFromPlatformRelease(pr *cores.PlatformRelease) Index {
 				URL:        pr.Platform.Package.URL,
 				Email:      pr.Platform.Package.Email,
 				Platforms: []*indexPlatformRelease{{
-					Name:                  pr.Platform.Name,
+					Name:                  pr.Name,
 					Architecture:          pr.Platform.Architecture,
 					Version:               pr.Version,
-					Deprecated:            pr.Platform.Deprecated,
-					Category:              pr.Platform.Category,
+					Deprecated:            pr.Deprecated,
+					Category:              pr.Category,
 					URL:                   pr.Resource.URL,
 					ArchiveFileName:       pr.Resource.ArchiveFileName,
 					Checksum:              pr.Resource.Checksum,
@@ -263,18 +263,13 @@ func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages, trust
 
 func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *cores.Package, trusted bool, isInstallJSON bool) error {
 	outPlatform := outPackage.GetOrCreatePlatform(inPlatformRelease.Architecture)
-	// FIXME: shall we use the Name and Category of the latest release? or maybe move Name and Category in PlatformRelease?
-	outPlatform.Name = inPlatformRelease.Name
-	outPlatform.Category = inPlatformRelease.Category
 	// If the variable `isInstallJSON` is false it means that the index we're reading is coming from the additional-urls.
 	// Therefore, the `outPlatform.Indexed` will be set at `true`.
 	outPlatform.Indexed = outPlatform.Indexed || !isInstallJSON
 
-	// If the Platform is installed before deprecation installed.json file does not include "deprecated" field.
-	// The installed.json is read during loading phase of an installed Platform, if the deprecated field is not found
-	// the package_index.json field would be overwritten and the deprecation info would be lost.
-	// This check prevents that behaviour.
-	if !outPlatform.Deprecated {
+	// If the latest platform release is deprecated, then deprecate the whole platform.
+	if outPlatform.Latest == nil || outPlatform.Latest.LessThan(inPlatformRelease.Version) {
+		outPlatform.Latest = inPlatformRelease.Version
 		outPlatform.Deprecated = inPlatformRelease.Deprecated
 	}
 
@@ -283,6 +278,8 @@ func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *core
 		return fmt.Errorf(tr("invalid platform archive size: %s"), err)
 	}
 	outPlatformRelease := outPlatform.GetOrCreateRelease(inPlatformRelease.Version)
+	outPlatformRelease.Name = inPlatformRelease.Name
+	outPlatformRelease.Category = inPlatformRelease.Category
 	outPlatformRelease.IsTrusted = trusted
 	outPlatformRelease.Resource = &resources.DownloadResource{
 		ArchiveFileName: inPlatformRelease.ArchiveFileName,
@@ -296,6 +293,7 @@ func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *core
 	outPlatformRelease.ToolDependencies = inPlatformRelease.extractToolDependencies()
 	outPlatformRelease.DiscoveryDependencies = inPlatformRelease.extractDiscoveryDependencies()
 	outPlatformRelease.MonitorDependencies = inPlatformRelease.extractMonitorDependencies()
+	outPlatformRelease.Deprecated = inPlatformRelease.Deprecated
 	return nil
 }
 

Diff for: arduino/cores/packageindex/index_test.go

@@ -91,11 +91,10 @@ func TestIndexFromPlatformRelease(t *testing.T) {
 				Name:     "serial-monitor",
 			},
 		},
+		Name:     "Arduino AVR Boards",
+		Category: "Arduino",
 		Platform: &cores.Platform{
-			Name:         "Arduino AVR Boards",
 			Architecture: "avr",
-			Category:     "Arduino",
-
 			Package: &cores.Package{
 				Name:       "arduino",
 				Maintainer: "Arduino",

Diff for: arduino/cores/packagemanager/loader.go

@@ -323,14 +323,14 @@ func (pm *Builder) loadPlatformRelease(platform *cores.PlatformRelease, path *pa
 		platform.Properties.Set("pluggable_monitor.required.serial", "builtin:serial-monitor")
 	}
 
-	if platform.Platform.Name == "" {
+	if platform.Name == "" {
 		if name, ok := platform.Properties.GetOk("name"); ok {
-			platform.Platform.Name = name
+			platform.Name = name
 		} else {
 			// If the platform.txt file doesn't exist for this platform and it's not in any
 			// package index there is no way of retrieving its name, so we build one using
 			// the available information, that is the packager name and the architecture.
-			platform.Platform.Name = fmt.Sprintf("%s-%s", platform.Platform.Package.Name, platform.Platform.Architecture)
+			platform.Name = fmt.Sprintf("%s-%s", platform.Platform.Package.Name, platform.Platform.Architecture)
 		}
 	}
 

Diff for: arduino/cores/packagemanager/package_manager.go

@@ -192,8 +192,8 @@ func (pme *Explorer) GetCustomGlobalProperties() *properties.Map {
 }
 
 // FindPlatformReleaseProvidingBoardsWithVidPid FIXMEDOC
-func (pme *Explorer) FindPlatformReleaseProvidingBoardsWithVidPid(vid, pid string) []*cores.PlatformRelease {
-	res := []*cores.PlatformRelease{}
+func (pme *Explorer) FindPlatformReleaseProvidingBoardsWithVidPid(vid, pid string) []*cores.Platform {
+	res := []*cores.Platform{}
 	for _, targetPackage := range pme.packages {
 		for _, targetPlatform := range targetPackage.Platforms {
 			platformRelease := targetPlatform.GetLatestRelease()
@@ -202,7 +202,7 @@ func (pme *Explorer) FindPlatformReleaseProvidingBoardsWithVidPid(vid, pid strin
 			}
 			for _, boardManifest := range platformRelease.BoardsManifest {
 				if boardManifest.HasUsbID(vid, pid) {
-					res = append(res, platformRelease)
+					res = append(res, targetPlatform)
 					break
 				}
 			}

Diff for: arduino/cores/packagemanager/profiles.go

@@ -45,7 +45,7 @@ func (pmb *Builder) LoadHardwareForProfile(p *sketch.Profile, installMissing boo
 			logrus.WithField("platform", platformRef).WithError(err).Debugf("Error loading platform for profile")
 		} else {
 			platformReleases = append(platformReleases, platformRelease)
-			indexURLs[platformRelease.Platform.Name] = platformRef.PlatformIndexURL
+			indexURLs[platformRelease.Name] = platformRef.PlatformIndexURL
 			logrus.WithField("platform", platformRef).Debugf("Loaded platform for profile")
 		}
 	}

Diff for: client_example/main.go

@@ -145,11 +145,6 @@ func main() {
 	log.Println("calling PlatformInstall(arduino:[email protected])")
 	callPlatformInstall(client, instance)
 
-	// Now list the installed platforms to double check previous installation
-	// went right.
-	log.Println("calling PlatformList()")
-	callPlatformList(client, instance)
-
 	// Upgrade the installed platform to the latest version.
 	log.Println("calling PlatformUpgrade(arduino:samd)")
 	callPlatformUpgrade(client, instance)
@@ -420,7 +415,7 @@ func callPlatformSearch(client rpc.ArduinoCoreServiceClient, instance *rpc.Insta
 	for _, plat := range platforms {
 		// We only print ID and version of the platforms found but you can look
 		// at the definition for the rpc.Platform struct for more fields.
-		log.Printf("Search result: %+v - %+v", plat.GetId(), plat.GetLatest())
+		log.Printf("Search result: %+v - %+v", plat.GetMetadata().GetId(), plat.GetLatestVersion())
 	}
 }
 
@@ -464,21 +459,6 @@ func callPlatformInstall(client rpc.ArduinoCoreServiceClient, instance *rpc.Inst
 	}
 }
 
-func callPlatformList(client rpc.ArduinoCoreServiceClient, instance *rpc.Instance) {
-	listResp, err := client.PlatformList(context.Background(),
-		&rpc.PlatformListRequest{Instance: instance})
-
-	if err != nil {
-		log.Fatalf("List error: %s", err)
-	}
-
-	for _, plat := range listResp.GetInstalledPlatforms() {
-		// We only print ID and version of the installed platforms but you can look
-		// at the definition for the rpc.Platform struct for more fields.
-		log.Printf("Installed platform: %s - %s", plat.GetId(), plat.GetInstalled())
-	}
-}
-
 func callPlatformUpgrade(client rpc.ArduinoCoreServiceClient, instance *rpc.Instance) {
 	upgradeRespStream, err := client.PlatformUpgrade(context.Background(),
 		&rpc.PlatformUpgradeRequest{
@@ -546,7 +526,7 @@ func callBoardSearch(client rpc.ArduinoCoreServiceClient, instance *rpc.Instance
 	}
 
 	for _, board := range res.Boards {
-		log.Printf("Board Name: %s, Board Platform: %s\n", board.Name, board.Platform.Id)
+		log.Printf("Board Name: %s, Board Platform: %s\n", board.Name, board.Platform.Metadata.Id)
 	}
 }
 

Diff for: commands/board/details.go

@@ -39,7 +39,7 @@ func Details(ctx context.Context, req *rpc.BoardDetailsRequest) (*rpc.BoardDetai
 		return nil, &arduino.InvalidFQBNError{Cause: err}
 	}
 
-	boardPackage, boardPlatform, board, boardProperties, boardRefPlatform, err := pme.ResolveFQBN(fqbn)
+	boardPackage, boardPlatformRelease, board, boardProperties, boardRefPlatform, err := pme.ResolveFQBN(fqbn)
 	if err != nil {
 		return nil, &arduino.UnknownFQBNError{Cause: err}
 	}
@@ -65,11 +65,11 @@ func Details(ctx context.Context, req *rpc.BoardDetailsRequest) (*rpc.BoardDetai
 	}
 
 	details.DebuggingSupported = boardProperties.ContainsKey("debug.executable") ||
-		boardPlatform.Properties.ContainsKey("debug.executable") ||
+		boardPlatformRelease.Properties.ContainsKey("debug.executable") ||
 		(boardRefPlatform != nil && boardRefPlatform.Properties.ContainsKey("debug.executable")) ||
 		// HOTFIX: Remove me when the `arduino:samd` core is updated
-		boardPlatform.String() == "arduino:[email protected]" ||
-		boardPlatform.String() == "arduino:[email protected]"
+		boardPlatformRelease.String() == "arduino:[email protected]" ||
+		boardPlatformRelease.String() == "arduino:[email protected]"
 
 	details.Package = &rpc.Package{
 		Name:       boardPackage.Name,
@@ -81,16 +81,16 @@ func Details(ctx context.Context, req *rpc.BoardDetailsRequest) (*rpc.BoardDetai
 	}
 
 	details.Platform = &rpc.BoardPlatform{
-		Architecture: boardPlatform.Platform.Architecture,
-		Category:     boardPlatform.Platform.Category,
-		Name:         boardPlatform.Platform.Name,
+		Architecture: boardPlatformRelease.Platform.Architecture,
+		Category:     boardPlatformRelease.Category,
+		Name:         boardPlatformRelease.Name,
 	}
 
-	if boardPlatform.Resource != nil {
-		details.Platform.Url = boardPlatform.Resource.URL
-		details.Platform.ArchiveFilename = boardPlatform.Resource.ArchiveFileName
-		details.Platform.Checksum = boardPlatform.Resource.Checksum
-		details.Platform.Size = boardPlatform.Resource.Size
+	if boardPlatformRelease.Resource != nil {
+		details.Platform.Url = boardPlatformRelease.Resource.URL
+		details.Platform.ArchiveFilename = boardPlatformRelease.Resource.ArchiveFileName
+		details.Platform.Checksum = boardPlatformRelease.Resource.Checksum
+		details.Platform.Size = boardPlatformRelease.Resource.Size
 	}
 
 	details.ConfigOptions = []*rpc.ConfigOption{}
@@ -118,7 +118,7 @@ func Details(ctx context.Context, req *rpc.BoardDetailsRequest) (*rpc.BoardDetai
 	}
 
 	details.ToolsDependencies = []*rpc.ToolsDependencies{}
-	for _, tool := range boardPlatform.ToolDependencies {
+	for _, tool := range boardPlatformRelease.ToolDependencies {
 		toolRelease := pme.FindToolDependency(tool)
 		var systems []*rpc.Systems
 		if toolRelease != nil {
@@ -141,9 +141,9 @@ func Details(ctx context.Context, req *rpc.BoardDetailsRequest) (*rpc.BoardDetai
 	}
 
 	details.Programmers = []*rpc.Programmer{}
-	for id, p := range boardPlatform.Programmers {
+	for id, p := range boardPlatformRelease.Programmers {
 		details.Programmers = append(details.Programmers, &rpc.Programmer{
-			Platform: boardPlatform.Platform.Name,
+			Platform: boardPlatformRelease.Name,
 			Id:       id,
 			Name:     p.Name,
 		})

Diff for: commands/board/list.go

@@ -157,7 +157,9 @@ func identify(pme *packagemanager.Explorer, port *discovery.Port) ([]*rpc.BoardL
 
 		// We need the Platform maintaner for sorting so we set it here
 		platform := &rpc.Platform{
-			Maintainer: board.PlatformRelease.Platform.Package.Maintainer,
+			Metadata: &rpc.PlatformMetadata{
+				Maintainer: board.PlatformRelease.Platform.Package.Maintainer,
+			},
 		}
 		boards = append(boards, &rpc.BoardListItem{
 			Name:     board.Name(),
@@ -185,7 +187,7 @@ func identify(pme *packagemanager.Explorer, port *discovery.Port) ([]*rpc.BoardL
 
 	// Put Arduino boards before others in case there are non Arduino boards with identical VID:PID combination
 	sort.SliceStable(boards, func(i, j int) bool {
-		if boards[i].Platform.Maintainer == "Arduino" && boards[j].Platform.Maintainer != "Arduino" {
+		if boards[i].Platform.Metadata.Maintainer == "Arduino" && boards[j].Platform.Metadata.Maintainer != "Arduino" {
 			return true
 		}
 		return false

Diff for: commands/board/listall.go

@@ -23,6 +23,7 @@ import (
 	"github.com/arduino/arduino-cli/arduino"
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/utils"
+	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/commands/internal/instances"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 )
@@ -46,29 +47,14 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllRequest) (*rpc.BoardListA
 				continue
 			}
 
-			installedVersion := installedPlatformRelease.Version.String()
-
-			latestVersion := ""
-			if latestPlatformRelease := platform.GetLatestRelease(); latestPlatformRelease != nil {
-				latestVersion = latestPlatformRelease.Version.String()
-			}
-
 			rpcPlatform := &rpc.Platform{
-				Id:                platform.String(),
-				Installed:         installedVersion,
-				Latest:            latestVersion,
-				Name:              platform.Name,
-				Maintainer:        platform.Package.Maintainer,
-				Website:           platform.Package.WebsiteURL,
-				Email:             platform.Package.Email,
-				ManuallyInstalled: platform.ManuallyInstalled,
-				Indexed:           platform.Indexed,
-				MissingMetadata:   !installedPlatformRelease.HasMetadata(),
+				Metadata: commands.PlatformToRPCPlatformMetadata(platform),
+				Release:  commands.PlatformReleaseToRPC(installedPlatformRelease),
 			}
 
 			toTest := []string{
 				platform.String(),
-				platform.Name,
+				installedPlatformRelease.Name,
 				platform.Architecture,
 				targetPackage.Name,
 				targetPackage.Maintainer,