go/internal/gcimporter: add "bundled" export data formats
Export data is self-contained, which is convenient for ensuring the
compiler only needs to open export data files directly referenced in
source file import declarations. But it does have a consequence that
export data files can redundantly encode information.
This CL adds a new "bundled" export data format that allows encoding
multiple complete packages together so they can share transitive
declarations. This is intended to be useful for storing a compilation
unit's source files along with any export data they depend upon for
efficient re-type-checking and analysis later.
Change-Id: I720a12cb051767f9d7ce95d645b448043a1e2167
Reviewed-on: https://go-review.googlesource.com/c/tools/+/292352
Trust: Matthew Dempsky <mdempsky@google.com>
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Rebecca Stambler <rstambler@golang.org>
diff --git a/go/gcexportdata/gcexportdata.go b/go/gcexportdata/gcexportdata.go
index 0e42514..04b65b6 100644
--- a/go/gcexportdata/gcexportdata.go
+++ b/go/gcexportdata/gcexportdata.go
@@ -105,3 +105,25 @@
}
return gcimporter.IExportData(out, fset, pkg)
}
+
+// ReadBundle reads an export bundle from in, decodes it, and returns type
+// information for the packages.
+// File position information is added to fset.
+//
+// ReadBundle may inspect and add to the imports map to ensure that references
+// within the export bundle to other packages are consistent.
+//
+// On return, the state of the reader is undefined.
+func ReadBundle(in io.Reader, fset *token.FileSet, imports map[string]*types.Package) ([]*types.Package, error) {
+ data, err := ioutil.ReadAll(in)
+ if err != nil {
+ return nil, fmt.Errorf("reading export bundle: %v", err)
+ }
+ return gcimporter.IImportBundle(fset, imports, data)
+}
+
+// WriteBundle writes encoded type information for the specified packages to out.
+// The FileSet provides file position information for named objects.
+func WriteBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
+ return gcimporter.IExportBundle(out, fset, pkgs)
+}
diff --git a/go/internal/gcimporter/iexport.go b/go/internal/gcimporter/iexport.go
index c4fae42..d2fc8b6 100644
--- a/go/internal/gcimporter/iexport.go
+++ b/go/internal/gcimporter/iexport.go
@@ -25,12 +25,25 @@
// 0: Go1.11 encoding
const iexportVersion = 0
+// Current bundled export format version. Increase with each format change.
+// 0: initial implementation
+const bundleVersion = 0
+
// IExportData writes indexed export data for pkg to out.
//
// If no file set is provided, position info will be missing.
// The package path of the top-level package will not be recorded,
// so that calls to IImportData can override with a provided package path.
-func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) (err error) {
+func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
+ return iexportCommon(out, fset, false, []*types.Package{pkg})
+}
+
+// IExportBundle writes an indexed export bundle for pkgs to out.
+func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
+ return iexportCommon(out, fset, true, pkgs)
+}
+
+func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*types.Package) (err error) {
defer func() {
if e := recover(); e != nil {
if ierr, ok := e.(internalError); ok {
@@ -48,7 +61,9 @@
stringIndex: map[string]uint64{},
declIndex: map[types.Object]uint64{},
typIndex: map[types.Type]uint64{},
- localpkg: pkg,
+ }
+ if !bundle {
+ p.localpkg = pkgs[0]
}
for i, pt := range predeclared() {
@@ -59,10 +74,20 @@
}
// Initialize work queue with exported declarations.
- scope := pkg.Scope()
- for _, name := range scope.Names() {
- if ast.IsExported(name) {
- p.pushDecl(scope.Lookup(name))
+ for _, pkg := range pkgs {
+ scope := pkg.Scope()
+ for _, name := range scope.Names() {
+ if ast.IsExported(name) {
+ p.pushDecl(scope.Lookup(name))
+ }
+ }
+
+ if bundle {
+ // Ensure pkg and its imports are included in the index.
+ p.allPkgs[pkg] = true
+ for _, imp := range pkg.Imports() {
+ p.allPkgs[imp] = true
+ }
}
}
@@ -75,10 +100,25 @@
dataLen := uint64(p.data0.Len())
w := p.newWriter()
w.writeIndex(p.declIndex)
+
+ if bundle {
+ w.uint64(uint64(len(pkgs)))
+ for _, pkg := range pkgs {
+ w.pkg(pkg)
+ imps := pkg.Imports()
+ w.uint64(uint64(len(imps)))
+ for _, imp := range imps {
+ w.pkg(imp)
+ }
+ }
+ }
w.flush()
// Assemble header.
var hdr intWriter
+ if bundle {
+ hdr.uint64(bundleVersion)
+ }
hdr.uint64(iexportVersion)
hdr.uint64(uint64(p.strings.Len()))
hdr.uint64(dataLen)
@@ -102,7 +142,9 @@
// For the main index, make sure to include every package that
// we reference, even if we're not exporting (or reexporting)
// any symbols from it.
- pkgObjs[w.p.localpkg] = nil
+ if w.p.localpkg != nil {
+ pkgObjs[w.p.localpkg] = nil
+ }
for pkg := range w.p.allPkgs {
pkgObjs[pkg] = nil
}
diff --git a/go/internal/gcimporter/iexport_test.go b/go/internal/gcimporter/iexport_test.go
index 8284ede..b6b4001 100644
--- a/go/internal/gcimporter/iexport_test.go
+++ b/go/internal/gcimporter/iexport_test.go
@@ -134,6 +134,21 @@
testPkgData(t, conf.Fset, pkg, exportdata)
}
}
+
+ var bundle bytes.Buffer
+ if err := gcimporter.IExportBundle(&bundle, conf.Fset, sorted); err != nil {
+ t.Fatal(err)
+ }
+ fset2 := token.NewFileSet()
+ imports := make(map[string]*types.Package)
+ pkgs2, err := gcimporter.IImportBundle(fset2, imports, bundle.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i, pkg := range sorted {
+ testPkg(t, conf.Fset, pkg, fset2, pkgs2[i])
+ }
}
func testPkgData(t *testing.T, fset *token.FileSet, pkg *types.Package, exportdata []byte) {
diff --git a/go/internal/gcimporter/iimport.go b/go/internal/gcimporter/iimport.go
index a31a880..b236deb 100644
--- a/go/internal/gcimporter/iimport.go
+++ b/go/internal/gcimporter/iimport.go
@@ -59,10 +59,23 @@
)
// IImportData imports a package from the serialized package data
-// and returns the number of bytes consumed and a reference to the package.
+// and returns 0 and a reference to the package.
// If the export data version is not recognized or the format is otherwise
// compromised, an error is returned.
-func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
+func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
+ pkgs, err := iimportCommon(fset, imports, data, false, path)
+ if err != nil {
+ return 0, nil, err
+ }
+ return 0, pkgs[0], nil
+}
+
+// IImportBundle imports a set of packages from the serialized package bundle.
+func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) {
+ return iimportCommon(fset, imports, data, true, "")
+}
+
+func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) {
const currentVersion = 1
version := int64(-1)
defer func() {
@@ -77,6 +90,15 @@
r := &intReader{bytes.NewReader(data), path}
+ if bundle {
+ bundleVersion := r.uint64()
+ switch bundleVersion {
+ case bundleVersion:
+ default:
+ errorf("unknown bundle format version %d", bundleVersion)
+ }
+ }
+
version = int64(r.uint64())
switch version {
case currentVersion, 0:
@@ -143,39 +165,58 @@
p.pkgIndex[pkg] = nameIndex
pkgList[i] = pkg
}
- if len(pkgList) == 0 {
- errorf("no packages found for %s", path)
- panic("unreachable")
+
+ if bundle {
+ pkgs = make([]*types.Package, r.uint64())
+ for i := range pkgs {
+ pkg := p.pkgAt(r.uint64())
+ imps := make([]*types.Package, r.uint64())
+ for j := range imps {
+ imps[j] = p.pkgAt(r.uint64())
+ }
+ pkg.SetImports(imps)
+ pkgs[i] = pkg
+ }
+ } else {
+ if len(pkgList) == 0 {
+ errorf("no packages found for %s", path)
+ panic("unreachable")
+ }
+ pkgs = pkgList[:1]
+
+ // record all referenced packages as imports
+ list := append(([]*types.Package)(nil), pkgList[1:]...)
+ sort.Sort(byPath(list))
+ pkgs[0].SetImports(list)
}
- p.ipkg = pkgList[0]
- names := make([]string, 0, len(p.pkgIndex[p.ipkg]))
- for name := range p.pkgIndex[p.ipkg] {
- names = append(names, name)
- }
- sort.Strings(names)
- for _, name := range names {
- p.doDecl(p.ipkg, name)
+
+ for _, pkg := range pkgs {
+ if pkg.Complete() {
+ continue
+ }
+
+ names := make([]string, 0, len(p.pkgIndex[pkg]))
+ for name := range p.pkgIndex[pkg] {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ for _, name := range names {
+ p.doDecl(pkg, name)
+ }
+
+ // package was imported completely and without errors
+ pkg.MarkComplete()
}
for _, typ := range p.interfaceList {
typ.Complete()
}
- // record all referenced packages as imports
- list := append(([]*types.Package)(nil), pkgList[1:]...)
- sort.Sort(byPath(list))
- p.ipkg.SetImports(list)
-
- // package was imported completely and without errors
- p.ipkg.MarkComplete()
-
- consumed, _ := r.Seek(0, io.SeekCurrent)
- return int(consumed), p.ipkg, nil
+ return pkgs, nil
}
type iimporter struct {
ipath string
- ipkg *types.Package
version int
stringData []byte
@@ -227,9 +268,6 @@
return pkg
}
path := p.stringAt(off)
- if path == p.ipath {
- return p.ipkg
- }
errorf("missing package %q in %q", path, p.ipath)
return nil
}
