go/internal/gcimporter: add "bundled" export data formats

Export data is self-contained, which is convenient for ensuring the
compiler only needs to open export data files directly referenced in
source file import declarations. But it does have a consequence that
export data files can redundantly encode information.

This CL adds a new "bundled" export data format that allows encoding
multiple complete packages together so they can share transitive
declarations. This is intended to be useful for storing a compilation
unit's source files along with any export data they depend upon for
efficient re-type-checking and analysis later.

Change-Id: I720a12cb051767f9d7ce95d645b448043a1e2167
Reviewed-on: https://go-review.googlesource.com/c/tools/+/292352
Trust: Matthew Dempsky <[email protected]>
Run-TryBot: Matthew Dempsky <[email protected]>
gopls-CI: kokoro <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Rebecca Stambler <[email protected]>

Author: mdempsky

go/gcexportdata/gcexportdata.go

@@ -105,3 +105,25 @@ func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
 	}
 	return gcimporter.IExportData(out, fset, pkg)
 }
+
+// ReadBundle reads an export bundle from in, decodes it, and returns type
+// information for the packages.
+// File position information is added to fset.
+//
+// ReadBundle may inspect and add to the imports map to ensure that references
+// within the export bundle to other packages are consistent.
+//
+// On return, the state of the reader is undefined.
+func ReadBundle(in io.Reader, fset *token.FileSet, imports map[string]*types.Package) ([]*types.Package, error) {
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return nil, fmt.Errorf("reading export bundle: %v", err)
+	}
+	return gcimporter.IImportBundle(fset, imports, data)
+}
+
+// WriteBundle writes encoded type information for the specified packages to out.
+// The FileSet provides file position information for named objects.
+func WriteBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
+	return gcimporter.IExportBundle(out, fset, pkgs)
+}

go/internal/gcimporter/iexport.go

@@ -25,12 +25,25 @@ import (
 // 0: Go1.11 encoding
 const iexportVersion = 0
 
+// Current bundled export format version. Increase with each format change.
+// 0: initial implementation
+const bundleVersion = 0
+
 // IExportData writes indexed export data for pkg to out.
 //
 // If no file set is provided, position info will be missing.
 // The package path of the top-level package will not be recorded,
 // so that calls to IImportData can override with a provided package path.
-func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) (err error) {
+func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
+	return iexportCommon(out, fset, false, []*types.Package{pkg})
+}
+
+// IExportBundle writes an indexed export bundle for pkgs to out.
+func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
+	return iexportCommon(out, fset, true, pkgs)
+}
+
+func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*types.Package) (err error) {
 	defer func() {
 		if e := recover(); e != nil {
 			if ierr, ok := e.(internalError); ok {
@@ -48,7 +61,9 @@ func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) (err er
 		stringIndex: map[string]uint64{},
 		declIndex:   map[types.Object]uint64{},
 		typIndex:    map[types.Type]uint64{},
-		localpkg:    pkg,
+	}
+	if !bundle {
+		p.localpkg = pkgs[0]
 	}
 
 	for i, pt := range predeclared() {
@@ -59,10 +74,20 @@ func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) (err er
 	}
 
 	// Initialize work queue with exported declarations.
-	scope := pkg.Scope()
-	for _, name := range scope.Names() {
-		if ast.IsExported(name) {
-			p.pushDecl(scope.Lookup(name))
+	for _, pkg := range pkgs {
+		scope := pkg.Scope()
+		for _, name := range scope.Names() {
+			if ast.IsExported(name) {
+				p.pushDecl(scope.Lookup(name))
+			}
+		}
+
+		if bundle {
+			// Ensure pkg and its imports are included in the index.
+			p.allPkgs[pkg] = true
+			for _, imp := range pkg.Imports() {
+				p.allPkgs[imp] = true
+			}
 		}
 	}
 
@@ -75,10 +100,25 @@ func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) (err er
 	dataLen := uint64(p.data0.Len())
 	w := p.newWriter()
 	w.writeIndex(p.declIndex)
+
+	if bundle {
+		w.uint64(uint64(len(pkgs)))
+		for _, pkg := range pkgs {
+			w.pkg(pkg)
+			imps := pkg.Imports()
+			w.uint64(uint64(len(imps)))
+			for _, imp := range imps {
+				w.pkg(imp)
+			}
+		}
+	}
 	w.flush()
 
 	// Assemble header.
 	var hdr intWriter
+	if bundle {
+		hdr.uint64(bundleVersion)
+	}
 	hdr.uint64(iexportVersion)
 	hdr.uint64(uint64(p.strings.Len()))
 	hdr.uint64(dataLen)
@@ -102,7 +142,9 @@ func (w *exportWriter) writeIndex(index map[types.Object]uint64) {
 	// For the main index, make sure to include every package that
 	// we reference, even if we're not exporting (or reexporting)
 	// any symbols from it.
-	pkgObjs[w.p.localpkg] = nil
+	if w.p.localpkg != nil {
+		pkgObjs[w.p.localpkg] = nil
+	}
 	for pkg := range w.p.allPkgs {
 		pkgObjs[pkg] = nil
 	}

go/internal/gcimporter/iexport_test.go

@@ -134,6 +134,21 @@ type UnknownType undefined
 			testPkgData(t, conf.Fset, pkg, exportdata)
 		}
 	}
+
+	var bundle bytes.Buffer
+	if err := gcimporter.IExportBundle(&bundle, conf.Fset, sorted); err != nil {
+		t.Fatal(err)
+	}
+	fset2 := token.NewFileSet()
+	imports := make(map[string]*types.Package)
+	pkgs2, err := gcimporter.IImportBundle(fset2, imports, bundle.Bytes())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i, pkg := range sorted {
+		testPkg(t, conf.Fset, pkg, fset2, pkgs2[i])
+	}
 }
 
 func testPkgData(t *testing.T, fset *token.FileSet, pkg *types.Package, exportdata []byte) {

go/internal/gcimporter/iimport.go

@@ -59,10 +59,23 @@ const (
 )
 
 // IImportData imports a package from the serialized package data
-// and returns the number of bytes consumed and a reference to the package.
+// and returns 0 and a reference to the package.
 // If the export data version is not recognized or the format is otherwise
 // compromised, an error is returned.
-func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
+func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
+	pkgs, err := iimportCommon(fset, imports, data, false, path)
+	if err != nil {
+		return 0, nil, err
+	}
+	return 0, pkgs[0], nil
+}
+
+// IImportBundle imports a set of packages from the serialized package bundle.
+func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) {
+	return iimportCommon(fset, imports, data, true, "")
+}
+
+func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) {
 	const currentVersion = 1
 	version := int64(-1)
 	defer func() {
@@ -77,6 +90,15 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 
 	r := &intReader{bytes.NewReader(data), path}
 
+	if bundle {
+		bundleVersion := r.uint64()
+		switch bundleVersion {
+		case bundleVersion:
+		default:
+			errorf("unknown bundle format version %d", bundleVersion)
+		}
+	}
+
 	version = int64(r.uint64())
 	switch version {
 	case currentVersion, 0:
@@ -143,39 +165,58 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		p.pkgIndex[pkg] = nameIndex
 		pkgList[i] = pkg
 	}
-	if len(pkgList) == 0 {
-		errorf("no packages found for %s", path)
-		panic("unreachable")
-	}
-	p.ipkg = pkgList[0]
-	names := make([]string, 0, len(p.pkgIndex[p.ipkg]))
-	for name := range p.pkgIndex[p.ipkg] {
-		names = append(names, name)
+
+	if bundle {
+		pkgs = make([]*types.Package, r.uint64())
+		for i := range pkgs {
+			pkg := p.pkgAt(r.uint64())
+			imps := make([]*types.Package, r.uint64())
+			for j := range imps {
+				imps[j] = p.pkgAt(r.uint64())
+			}
+			pkg.SetImports(imps)
+			pkgs[i] = pkg
+		}
+	} else {
+		if len(pkgList) == 0 {
+			errorf("no packages found for %s", path)
+			panic("unreachable")
+		}
+		pkgs = pkgList[:1]
+
+		// record all referenced packages as imports
+		list := append(([]*types.Package)(nil), pkgList[1:]...)
+		sort.Sort(byPath(list))
+		pkgs[0].SetImports(list)
 	}
-	sort.Strings(names)
-	for _, name := range names {
-		p.doDecl(p.ipkg, name)
+
+	for _, pkg := range pkgs {
+		if pkg.Complete() {
+			continue
+		}
+
+		names := make([]string, 0, len(p.pkgIndex[pkg]))
+		for name := range p.pkgIndex[pkg] {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		for _, name := range names {
+			p.doDecl(pkg, name)
+		}
+
+		// package was imported completely and without errors
+		pkg.MarkComplete()
 	}
 
 	for _, typ := range p.interfaceList {
 		typ.Complete()
 	}
 
-	// record all referenced packages as imports
-	list := append(([]*types.Package)(nil), pkgList[1:]...)
-	sort.Sort(byPath(list))
-	p.ipkg.SetImports(list)
-
-	// package was imported completely and without errors
-	p.ipkg.MarkComplete()
-
-	consumed, _ := r.Seek(0, io.SeekCurrent)
-	return int(consumed), p.ipkg, nil
+	return pkgs, nil
 }
 
 type iimporter struct {
 	ipath   string
-	ipkg    *types.Package
 	version int
 
 	stringData  []byte
@@ -227,9 +268,6 @@ func (p *iimporter) pkgAt(off uint64) *types.Package {
 		return pkg
 	}
 	path := p.stringAt(off)
-	if path == p.ipath {
-		return p.ipkg
-	}
 	errorf("missing package %q in %q", path, p.ipath)
 	return nil
 }