cmd/go: record origin metadata during module download

This change adds an "Origin" JSON key to the output of
go list -json -m and go mod download -json. The associated value is a
JSON object with metadata about the source control system. For Git,
that metadata is sufficient to evaluate whether the remote server has
changed in any interesting way that might invalidate the cached data.
In most cases, it will not have, and a fetch could then avoid
downloading a full repo from the server.

This origin metadata is also now recorded in the .info file for a
given module@version, for informational and debugging purposes.

This change only adds the metadata. It does not use it to optimize
away unnecessary git fetch operations. (That's the next change.)

For #53644.

Change-Id: I4a1712a2386d1d8ab4e02ffdf0f72ba75d556115
Reviewed-on: https://go-review.googlesource.com/c/go/+/411397
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Bryan Mills <[email protected]>

Author: rsc

src/cmd/go/internal/modcmd/download.go

@@ -149,7 +149,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 
 	downloadModule := func(m *moduleJSON) {
 		var err error
-		m.Info, err = modfetch.InfoFile(m.Path, m.Version)
+		_, m.Info, err = modfetch.InfoFile(m.Path, m.Version)
 		if err != nil {
 			m.Error = err.Error()
 			return

src/cmd/go/internal/modfetch/cache.go

@@ -164,7 +164,7 @@ func SideLock() (unlock func(), err error) {
 }
 
 // A cachingRepo is a cache around an underlying Repo,
-// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
+// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip).
 // It is also safe for simultaneous use by multiple goroutines
 // (so that it can be returned from Lookup multiple times).
 // It serializes calls to the underlying Repo.
@@ -195,24 +195,32 @@ func (r *cachingRepo) repo() Repo {
 	return r.r
 }
 
+func (r *cachingRepo) CheckReuse(old *codehost.Origin) error {
+	return r.repo().CheckReuse(old)
+}
+
 func (r *cachingRepo) ModulePath() string {
 	return r.path
 }
 
-func (r *cachingRepo) Versions(prefix string) ([]string, error) {
+func (r *cachingRepo) Versions(prefix string) (*Versions, error) {
 	type cached struct {
-		list []string
-		err  error
+		v   *Versions
+		err error
 	}
 	c := r.cache.Do("versions:"+prefix, func() any {
-		list, err := r.repo().Versions(prefix)
-		return cached{list, err}
+		v, err := r.repo().Versions(prefix)
+		return cached{v, err}
 	}).(cached)
 
 	if c.err != nil {
 		return nil, c.err
 	}
-	return append([]string(nil), c.list...), nil
+	v := &Versions{
+		Origin: c.v.Origin,
+		List:   append([]string(nil), c.v.List...),
+	}
+	return v, nil
 }
 
 type cachedInfo struct {
@@ -310,31 +318,35 @@ func (r *cachingRepo) Zip(dst io.Writer, version string) error {
 	return r.repo().Zip(dst, version)
 }
 
-// InfoFile is like Lookup(path).Stat(version) but returns the name of the file
+// InfoFile is like Lookup(path).Stat(version) but also returns the name of the file
 // containing the cached information.
-func InfoFile(path, version string) (string, error) {
+func InfoFile(path, version string) (*RevInfo, string, error) {
 	if !semver.IsValid(version) {
-		return "", fmt.Errorf("invalid version %q", version)
+		return nil, "", fmt.Errorf("invalid version %q", version)
 	}
 
-	if file, _, err := readDiskStat(path, version); err == nil {
-		return file, nil
+	if file, info, err := readDiskStat(path, version); err == nil {
+		return info, file, nil
 	}
 
+	var info *RevInfo
 	err := TryProxies(func(proxy string) error {
-		_, err := Lookup(proxy, path).Stat(version)
+		i, err := Lookup(proxy, path).Stat(version)
+		if err == nil {
+			info = i
+		}
 		return err
 	})
 	if err != nil {
-		return "", err
+		return nil, "", err
 	}
 
 	// Stat should have populated the disk cache for us.
 	file, err := CachePath(module.Version{Path: path, Version: version}, "info")
 	if err != nil {
-		return "", err
+		return nil, "", err
 	}
-	return file, nil
+	return info, file, nil
 }
 
 // GoMod is like Lookup(path).GoMod(rev) but avoids the

src/cmd/go/internal/modfetch/codehost/codehost.go

@@ -22,6 +22,9 @@ import (
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/lockedfile"
 	"cmd/go/internal/str"
+
+	"golang.org/x/mod/module"
+	"golang.org/x/mod/semver"
 )
 
 // Downloaded size limits.
@@ -36,8 +39,15 @@ const (
 // remote version control servers, and code hosting sites.
 // A Repo must be safe for simultaneous use by multiple goroutines.
 type Repo interface {
+	// CheckReuse checks whether the old origin information
+	// remains up to date. If so, whatever cached object it was
+	// taken from can be reused.
+	// The subdir gives subdirectory name where the module root is expected to be found,
+	// "" for the root or "sub/dir" for a subdirectory (no trailing slash).
+	CheckReuse(old *Origin, subdir string) error
+
 	// List lists all tags with the given prefix.
-	Tags(prefix string) (tags []string, err error)
+	Tags(prefix string) (*Tags, error)
 
 	// Stat returns information about the revision rev.
 	// A revision can be any identifier known to the underlying service:
@@ -74,8 +84,84 @@ type Repo interface {
 	DescendsFrom(rev, tag string) (bool, error)
 }
 
-// A Rev describes a single revision in a source code repository.
+// An Origin describes the provenance of a given repo method result.
+// It can be passed to CheckReuse (usually in a different go command invocation)
+// to see whether the result remains up-to-date.
+type Origin struct {
+	VCS    string `json:",omitempty"` // "git" etc
+	URL    string `json:",omitempty"` // URL of repository
+	Subdir string `json:",omitempty"` // subdirectory in repo
+
+	// If TagSum is non-empty, then the resolution of this module version
+	// depends on the set of tags present in the repo, specifically the tags
+	// of the form TagPrefix + a valid semver version.
+	// If the matching repo tags and their commit hashes still hash to TagSum,
+	// the Origin is still valid (at least as far as the tags are concerned).
+	// The exact checksum is up to the Repo implementation; see (*gitRepo).Tags.
+	TagPrefix string `json:",omitempty"`
+	TagSum    string `json:",omitempty"`
+
+	// If Ref is non-empty, then the resolution of this module version
+	// depends on Ref resolving to the revision identified by Hash.
+	// If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned).
+	// For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3",
+	// and the Hash is the Git object hash the ref maps to.
+	// Other VCS might choose differently, but the idea is that Ref is the name
+	// with a mutable meaning while Hash is a name with an immutable meaning.
+	Ref  string `json:",omitempty"`
+	Hash string `json:",omitempty"`
+}
+
+// Checkable reports whether the Origin contains anything that can be checked.
+// If not, it's purely informational and should fail a CheckReuse call.
+func (o *Origin) Checkable() bool {
+	return o.TagSum != "" || o.Ref != "" || o.Hash != ""
+}
+
+func (o *Origin) Merge(other *Origin) {
+	if o.TagSum == "" {
+		o.TagPrefix = other.TagPrefix
+		o.TagSum = other.TagSum
+	}
+	if o.Ref == "" {
+		o.Ref = other.Ref
+		o.Hash = other.Hash
+	}
+}
+
+// A Tags describes the available tags in a code repository.
+type Tags struct {
+	Origin *Origin
+	List   []Tag
+}
+
+// A Tag describes a single tag in a code repository.
+type Tag struct {
+	Name string
+	Hash string // content hash identifying tag's content, if available
+}
+
+// isOriginTag reports whether tag should be preserved
+// in the Tags method's Origin calculation.
+// We can safely ignore tags that are not look like pseudo-versions,
+// because ../coderepo.go's (*codeRepo).Versions ignores them too.
+// We can also ignore non-semver tags, but we have to include semver
+// tags with extra suffixes, because the pseudo-version base finder uses them.
+func isOriginTag(tag string) bool {
+	// modfetch.(*codeRepo).Versions uses Canonical == tag,
+	// but pseudo-version calculation has a weaker condition that
+	// the canonical is a prefix of the tag.
+	// Include those too, so that if any new one appears, we'll invalidate the cache entry.
+	// This will lead to spurious invalidation of version list results,
+	// but tags of this form being created should be fairly rare
+	// (and invalidate pseudo-version results anyway).
+	c := semver.Canonical(tag)
+	return c != "" && strings.HasPrefix(tag, c) && !module.IsPseudoVersion(tag)
+}
+
+// A RevInfo describes a single revision in a source code repository.
 type RevInfo struct {
+	Origin  *Origin
 	Name    string    // complete ID in underlying repository
 	Short   string    // shortened ID, for use in pseudo-version
 	Version string    // version used in lookup