cmd/go: add -reuse flag to make proxy invocations more efficient

The go list -m and go mod download commands now have a -reuse flag,
which is passed the name of a file containing the JSON output from a
previous run of the same command. (It is up to the caller to ensure
that flags such as -versions or -retracted, which affect the output,
are consistent between the old and new run.)

The new run uses the old JSON to evaluate whether the answer is
unchanged since the old run. If so, it reuses that information,
avoiding a costly 'git fetch', and sets a new Reuse: true field in its
own JSON output.

This dance with saving the JSON output and passing it back to -reuse
is not necessary on most systems, because the go command caches
version control checkouts in the module cache. That cache means that a
new 'git fetch' would only download the commits that are new since the
previous one (often none at all).

The dance becomes important only on systems that do not preserve the
module cache, for example by running 'go clean -modcache' aggressively
or by running in some environment that starts with an empty file
system.

For golang#53644.

Change-Id: I447960abf8055f83cc6dbc699a9fde9931130004
Reviewed-on: https://go-review.googlesource.com/c/go/+/411398
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Bryan Mills <[email protected]>

Author: rsc

src/cmd/go/alldocs.go

@@ -223,6 +223,7 @@ applied to a Go struct, but now a Module struct:
 
     type Module struct {
         Path       string        // module path
+        Query      string        // version query corresponding to this version
         Version    string        // module version
         Versions   []string      // available module versions
         Replace    *Module       // replaced by this module
@@ -236,6 +237,8 @@ applied to a Go struct, but now a Module struct:
         Retracted  []string      // retraction information, if any (with -retracted or -u)
         Deprecated string        // deprecation message, if any (with -u)
         Error      *ModuleError  // error loading module
+        Origin     any           // provenance of module
+        Reuse      bool          // reuse of old module info is safe
     }
 
     type ModuleError struct {
@@ -312,6 +315,16 @@ that must be a module path or query and returns the specified
 module as a Module struct. If an error occurs, the result will
 be a Module struct with a non-nil Error field.
 
+When using -m, the -reuse=old.json flag accepts the name of file containing
+the JSON output of a previous 'go list -m -json' invocation with the
+same set of modifier flags (such as -u, -retracted, and -versions).
+The go command may use this file to determine that a module is unchanged
+since the previous invocation and avoid redownloading information about it.
+Modules that are not redownloaded will be marked in the new output by
+setting the Reuse field to true. Normally the module cache provides this
+kind of reuse automatically; the -reuse flag can be useful on systems that
+do not preserve the module cache.
+
 For more about build flags, see 'go help build'.
 
 For more about specifying packages, see 'go help packages'.
@@ -337,6 +350,7 @@ var (
 	listJsonFields jsonFlag // If not empty, only output these fields.
 	listM          = CmdList.Flag.Bool("m", false, "")
 	listRetracted  = CmdList.Flag.Bool("retracted", false, "")
+	listReuse      = CmdList.Flag.String("reuse", "", "")
 	listTest       = CmdList.Flag.Bool("test", false, "")
 	listU          = CmdList.Flag.Bool("u", false, "")
 	listVersions   = CmdList.Flag.Bool("versions", false, "")
@@ -398,6 +412,12 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 	if *listFmt != "" && listJson == true {
 		base.Fatalf("go list -f cannot be used with -json")
 	}
+	if *listReuse != "" && !*listM {
+		base.Fatalf("go list -reuse cannot be used without -m")
+	}
+	if *listReuse != "" && modload.HasModRoot() {
+		base.Fatalf("go list -reuse cannot be used inside a module")
+	}
 
 	work.BuildInit()
 	out := newTrackingWriter(os.Stdout)
@@ -532,7 +552,10 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 				mode |= modload.ListRetractedVersions
 			}
 		}
-		mods, err := modload.ListModules(ctx, args, mode)
+		if *listReuse != "" && len(args) == 0 {
+			base.Fatalf("go: list -m -reuse only has an effect with module@version arguments")
+		}
+		mods, err := modload.ListModules(ctx, args, mode, *listReuse)
 		if !*listE {
 			for _, m := range mods {
 				if m.Error != nil {
@@ -783,7 +806,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 			if *listRetracted {
 				mode |= modload.ListRetracted
 			}
-			rmods, err := modload.ListModules(ctx, args, mode)
+			rmods, err := modload.ListModules(ctx, args, mode, *listReuse)
 			if err != nil && !*listE {
 				base.Errorf("go: %v", err)
 			}

src/cmd/go/internal/list/list.go

@@ -13,14 +13,15 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/modfetch"
+	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/modload"
 
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/semver"
 )
 
 var cmdDownload = &base.Command{
-	UsageLine: "go mod download [-x] [-json] [modules]",
+	UsageLine: "go mod download [-x] [-json] [-reuse=old.json] [modules]",
 	Short:     "download modules to local cache",
 	Long: `
 Download downloads the named modules, which can be module patterns selecting
@@ -44,6 +45,7 @@ corresponding to this Go struct:
 
     type Module struct {
         Path     string // module path
+        Query    string // version query corresponding to this version
         Version  string // module version
         Error    string // error loading module
         Info     string // absolute path to cached .info file
@@ -52,8 +54,18 @@ corresponding to this Go struct:
         Dir      string // absolute path to cached source root directory
         Sum      string // checksum for path, version (as in go.sum)
         GoModSum string // checksum for go.mod (as in go.sum)
+        Origin   any    // provenance of module
+        Reuse    bool   // reuse of old module info is safe
     }
 
+The -reuse flag accepts the name of file containing the JSON output of a
+previous 'go mod download -json' invocation. The go command may use this
+file to determine that a module is unchanged since the previous invocation
+and avoid redownloading it. Modules that are not redownloaded will be marked
+in the new output by setting the Reuse field to true. Normally the module
+cache provides this kind of reuse automatically; the -reuse flag can be
+useful on systems that do not preserve the module cache.
+
 The -x flag causes download to print the commands download executes.
 
 See https://golang.org/ref/mod#go-mod-download for more about 'go mod download'.
@@ -62,7 +74,10 @@ See https://golang.org/ref/mod#version-queries for more about version queries.
 	`,
 }
 
-var downloadJSON = cmdDownload.Flag.Bool("json", false, "")
+var (
+	downloadJSON  = cmdDownload.Flag.Bool("json", false, "")
+	downloadReuse = cmdDownload.Flag.String("reuse", "", "")
+)
 
 func init() {
 	cmdDownload.Run = runDownload // break init cycle
@@ -75,13 +90,17 @@ func init() {
 type moduleJSON struct {
 	Path     string `json:",omitempty"`
 	Version  string `json:",omitempty"`
+	Query    string `json:",omitempty"`
 	Error    string `json:",omitempty"`
 	Info     string `json:",omitempty"`
 	GoMod    string `json:",omitempty"`
 	Zip      string `json:",omitempty"`
 	Dir      string `json:",omitempty"`
 	Sum      string `json:",omitempty"`
 	GoModSum string `json:",omitempty"`
+
+	Origin *codehost.Origin `json:",omitempty"`
+	Reuse  bool             `json:",omitempty"`
 }
 
 func runDownload(ctx context.Context, cmd *base.Command, args []string) {
@@ -148,12 +167,12 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 	}
 
 	downloadModule := func(m *moduleJSON) {
-		var err error
-		_, m.Info, err = modfetch.InfoFile(m.Path, m.Version)
+		_, file, err := modfetch.InfoFile(m.Path, m.Version)
 		if err != nil {
 			m.Error = err.Error()
 			return
 		}
+		m.Info = file
 		m.GoMod, err = modfetch.GoModFile(m.Path, m.Version)
 		if err != nil {
 			m.Error = err.Error()
@@ -179,9 +198,14 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 	}
 
 	var mods []*moduleJSON
+
+	if *downloadReuse != "" && modload.HasModRoot() {
+		base.Fatalf("go mod download -reuse cannot be used inside a module")
+	}
+
 	type token struct{}
 	sem := make(chan token, runtime.GOMAXPROCS(0))
-	infos, infosErr := modload.ListModules(ctx, args, 0)
+	infos, infosErr := modload.ListModules(ctx, args, 0, *downloadReuse)
 	if !haveExplicitArgs {
 		// 'go mod download' is sometimes run without arguments to pre-populate the
 		// module cache. It may fetch modules that aren't needed to build packages
@@ -209,12 +233,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 		m := &moduleJSON{
 			Path:    info.Path,
 			Version: info.Version,
+			Query:   info.Query,
+			Reuse:   info.Reuse,
+			Origin:  info.Origin,
 		}
 		mods = append(mods, m)
 		if info.Error != nil {
 			m.Error = info.Error.Err
 			continue
 		}
+		if m.Reuse {
+			continue
+		}
 		sem <- token{}
 		go func() {
 			downloadModule(m)

src/cmd/go/internal/modcmd/download.go

@@ -82,7 +82,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) {
 			}
 		}
 
-		mods, err := modload.ListModules(ctx, args, 0)
+		mods, err := modload.ListModules(ctx, args, 0, "")
 		if err != nil {
 			base.Fatalf("go: %v", err)
 		}

src/cmd/go/internal/modcmd/why.go

@@ -573,6 +573,26 @@ func writeDiskStat(file string, info *RevInfo) error {
 	if file == "" {
 		return nil
 	}
+
+	if info.Origin != nil {
+		// Clean the origin information, which might have too many
+		// validation criteria, for example if we are saving the result of
+		// m@master as m@pseudo-version.
+		clean := *info
+		info = &clean
+		o := *info.Origin
+		info.Origin = &o
+
+		// Tags never matter if you are starting with a semver version,
+		// as we would be when finding this cache entry.
+		o.TagSum = ""
+		o.TagPrefix = ""
+		// Ref doesn't matter if you have a pseudoversion.
+		if module.IsPseudoVersion(info.Version) {
+			o.Ref = ""
+		}
+	}
+
 	js, err := json.Marshal(info)
 	if err != nil {
 		return err

src/cmd/go/internal/modfetch/cache.go

@@ -113,20 +113,17 @@ type Origin struct {
 }
 
 // Checkable reports whether the Origin contains anything that can be checked.
-// If not, it's purely informational and should fail a CheckReuse call.
+// If not, the Origin is purely informational and should fail a CheckReuse call.
 func (o *Origin) Checkable() bool {
 	return o.TagSum != "" || o.Ref != "" || o.Hash != ""
 }
 
-func (o *Origin) Merge(other *Origin) {
-	if o.TagSum == "" {
-		o.TagPrefix = other.TagPrefix
-		o.TagSum = other.TagSum
-	}
-	if o.Ref == "" {
-		o.Ref = other.Ref
-		o.Hash = other.Hash
-	}
+// ClearCheckable clears the Origin enough to make Checkable return false.
+func (o *Origin) ClearCheckable() {
+	o.TagSum = ""
+	o.TagPrefix = ""
+	o.Ref = ""
+	o.Hash = ""
 }
 
 // A Tags describes the available tags in a code repository.

src/cmd/go/internal/modfetch/codehost/codehost.go

@@ -423,8 +423,11 @@ func (r *gitRepo) stat(rev string) (info *RevInfo, err error) {
 
 	defer func() {
 		if info != nil {
-			info.Origin.Ref = ref
 			info.Origin.Hash = info.Name
+			// There's a ref = hash below; don't write that hash down as Origin.Ref.
+			if ref != info.Origin.Hash {
+				info.Origin.Ref = ref
+			}
 		}
 	}()
 

src/cmd/go/internal/modfetch/codehost/git.go

@@ -153,6 +153,9 @@ func (r *codeRepo) Versions(prefix string) (*Versions, error) {
 			Err:  err,
 		}
 	}
+	if tags.Origin != nil {
+		tags.Origin.Subdir = r.codeDir
+	}
 
 	var list, incompatible []string
 	for _, tag := range tags.List {
@@ -450,23 +453,26 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 		}
 
 		origin := info.Origin
-		if module.IsPseudoVersion(v) {
-			// Add tags that are relevant to pseudo-version calculation to origin.
-			prefix := ""
-			if r.codeDir != "" {
-				prefix = r.codeDir + "/"
-			}
-			if r.pathMajor != "" { // "/v2" or "/.v2"
-				prefix += r.pathMajor[1:] + "." // += "v2."
-			}
-			tags, err := r.code.Tags(prefix)
-			if err != nil {
-				return nil, err
-			}
+		if origin != nil {
 			o := *origin
 			origin = &o
-			origin.TagPrefix = tags.Origin.TagPrefix
-			origin.TagSum = tags.Origin.TagSum
+			origin.Subdir = r.codeDir
+			if module.IsPseudoVersion(v) && (v != statVers || !strings.HasPrefix(v, "v0.0.0-")) {
+				// Add tags that are relevant to pseudo-version calculation to origin.
+				prefix := r.codeDir
+				if prefix != "" {
+					prefix += "/"
+				}
+				if r.pathMajor != "" { // "/v2" or "/.v2"
+					prefix += r.pathMajor[1:] + "." // += "v2."
+				}
+				tags, err := r.code.Tags(prefix)
+				if err != nil {
+					return nil, err
+				}
+				origin.TagPrefix = tags.Origin.TagPrefix
+				origin.TagSum = tags.Origin.TagSum
+			}
 		}
 
 		return &RevInfo{