cmd/go/internal/vgo: load packages in parallel
This is a large CL. The best way to read it is probably to treat
vgo/load.go as if it were a newly-written file, not by reading diffs.
The main motivation here is to do package loading in parallel,
including the on-demand downloading of dependency zip files.
Running the load in parallel requires making the structure a bit
cleaner than it was before: the old recursive traversal could
retraverse sections of the graph if it decided that now it needed
tests and their dependencies, and so on. To parallelize, the work
had to be split up into items visited exactly once, and the result
is a bit cleaner.
Most of vgo/search.go has been absorbed into load.go's ImportPaths.
Previously, there was an odd recursion where ImportPaths's loader
callback could call into MatchAll/MatchALL, which created and ran
a new loader before returning back to the original loader's callback.
The new structure more properly expresses the dependency of
the loader on matching, which in turn depends on the current state
of the loader (namely buildList).
The magic "ALL" package pattern is gone. Code that needs that
functionality now has a proper entry point (vgo.LoadALL).
Finally, isolate all the "what happened?" loader state consulted
after the fact into a single struct. The build list remains separate,
at least for now.
The cleaner structure sets us up well for the next CL.
Fixes golang/go#25162.
Change-Id: I963f0effd7314f7e77eaad058774dbd2586c9cd2
Reviewed-on: https://go-review.googlesource.com/121303
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/vendor/cmd/go/internal/load/pkg.go b/vendor/cmd/go/internal/load/pkg.go
index 41ebf89..8ccfb77 100644
--- a/vendor/cmd/go/internal/load/pkg.go
+++ b/vendor/cmd/go/internal/load/pkg.go
@@ -35,7 +35,7 @@
VgoImportPaths func(args []string) []string // expand import paths
VgoPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
VgoModInfoProg func(info string) []byte // wrap module info in .go code for binary
- VgoAddImports func([]string) // update go.mod to add modules for imports in these files
+ VgoImportFromFiles func([]string) // update go.mod to add modules for imports in these files
)
var IgnoreImports bool // control whether we ignore imports in packages
@@ -1803,8 +1803,8 @@
}
ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil }
- if VgoAddImports != nil {
- VgoAddImports(gofiles)
+ if VgoImportFromFiles != nil {
+ VgoImportFromFiles(gofiles)
}
var err error
diff --git a/vendor/cmd/go/internal/modcmd/mod.go b/vendor/cmd/go/internal/modcmd/mod.go
index 554b695..991f4e7 100644
--- a/vendor/cmd/go/internal/modcmd/mod.go
+++ b/vendor/cmd/go/internal/modcmd/mod.go
@@ -250,12 +250,12 @@
if *modSync || *modVendor || needBuildList {
var pkgs []string
if *modSync || *modVendor {
- pkgs = vgo.ImportPaths([]string{"ALL"})
+ pkgs = vgo.LoadALL()
} else {
vgo.LoadBuildList()
}
if *modSync {
- // ImportPaths(ALL) already added missing modules.
+ // LoadALL already added missing modules.
// Remove unused modules.
used := map[module.Version]bool{vgo.Target: true}
for _, pkg := range pkgs {
diff --git a/vendor/cmd/go/internal/modcmd/vendor.go b/vendor/cmd/go/internal/modcmd/vendor.go
index 441aacf..6dc1f4a 100644
--- a/vendor/cmd/go/internal/modcmd/vendor.go
+++ b/vendor/cmd/go/internal/modcmd/vendor.go
@@ -21,7 +21,7 @@
var copiedDir map[string]bool
func runVendor() {
- pkgs := vgo.ImportPaths([]string{"ALL"})
+ pkgs := vgo.LoadALL()
vdir := filepath.Join(vgo.ModRoot, "vendor")
if err := os.RemoveAll(vdir); err != nil {
diff --git a/vendor/cmd/go/internal/modfetch/fetch.go b/vendor/cmd/go/internal/modfetch/fetch.go
index 657f0d5..f4eb651 100644
--- a/vendor/cmd/go/internal/modfetch/fetch.go
+++ b/vendor/cmd/go/internal/modfetch/fetch.go
@@ -19,37 +19,50 @@
"cmd/go/internal/base"
"cmd/go/internal/dirhash"
"cmd/go/internal/module"
+ "cmd/go/internal/par"
)
+var downloadCache par.Cache
+
// Download downloads the specific module version to the
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
func Download(mod module.Version) (dir string, err error) {
- modpath := mod.Path + "@" + mod.Version
- dir = filepath.Join(SrcMod, modpath)
- if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
- zipfile := filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".zip")
- if _, err := os.Stat(zipfile); err == nil {
- // Use it.
- // This should only happen if the mod/cache directory is preinitialized
- // or if src/mod/path was removed but not src/mod/cache/download.
- fmt.Fprintf(os.Stderr, "vgo: extracting %s %s\n", mod.Path, mod.Version)
- } else {
- if err := os.MkdirAll(filepath.Join(SrcMod, "cache/download", mod.Path, "@v"), 0777); err != nil {
- return "", err
- }
- fmt.Fprintf(os.Stderr, "vgo: downloading %s %s\n", mod.Path, mod.Version)
- if err := downloadZip(mod, zipfile); err != nil {
- return "", err
- }
- }
- if err := Unzip(dir, zipfile, modpath, 0); err != nil {
- fmt.Fprintf(os.Stderr, "-> %s\n", err)
- return "", err
- }
+ // The par.Cache here avoids duplicate work but also
+ // avoids conflicts from simultaneous calls by multiple goroutines
+ // for the same version.
+ type cached struct {
+ dir string
+ err error
}
- checkSum(mod)
- return dir, nil
+ c := downloadCache.Do(mod, func() interface{} {
+ modpath := mod.Path + "@" + mod.Version
+ dir = filepath.Join(SrcMod, modpath)
+ if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
+ zipfile := filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".zip")
+ if _, err := os.Stat(zipfile); err == nil {
+ // Use it.
+ // This should only happen if the mod/cache directory is preinitialized
+ // or if src/mod/path was removed but not src/mod/cache/download.
+ fmt.Fprintf(os.Stderr, "vgo: extracting %s %s\n", mod.Path, mod.Version)
+ } else {
+ if err := os.MkdirAll(filepath.Join(SrcMod, "cache/download", mod.Path, "@v"), 0777); err != nil {
+ return cached{"", err}
+ }
+ fmt.Fprintf(os.Stderr, "vgo: downloading %s %s\n", mod.Path, mod.Version)
+ if err := downloadZip(mod, zipfile); err != nil {
+ return cached{"", err}
+ }
+ }
+ if err := Unzip(dir, zipfile, modpath, 0); err != nil {
+ fmt.Fprintf(os.Stderr, "-> %s\n", err)
+ return cached{"", err}
+ }
+ }
+ checkSum(mod)
+ return cached{dir, nil}
+ }).(cached)
+ return c.dir, c.err
}
func downloadZip(mod module.Version, target string) error {
diff --git a/vendor/cmd/go/internal/modfetch/unzip.go b/vendor/cmd/go/internal/modfetch/unzip.go
index 9d9e298..2e08aaa 100644
--- a/vendor/cmd/go/internal/modfetch/unzip.go
+++ b/vendor/cmd/go/internal/modfetch/unzip.go
@@ -77,7 +77,6 @@
}
r, err := zf.Open()
if err != nil {
- r.Close()
w.Close()
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
diff --git a/vendor/cmd/go/internal/par/work.go b/vendor/cmd/go/internal/par/work.go
index 44a4e8c..a568c86 100644
--- a/vendor/cmd/go/internal/par/work.go
+++ b/vendor/cmd/go/internal/par/work.go
@@ -8,6 +8,7 @@
import (
"math/rand"
"sync"
+ "sync/atomic"
)
// Work manages a set of work items to be executed in parallel, at most once each.
@@ -106,23 +107,43 @@
m sync.Map
}
+type cacheEntry struct {
+ done uint32
+ mu sync.Mutex
+ result interface{}
+}
+
// Do calls the function f if and only if Do is being called for the first time with this key.
// No call to Do with a given key returns until the one call to f returns.
// Do returns the value returned by the one call to f.
func (c *Cache) Do(key interface{}, f func() interface{}) interface{} {
- type entry struct {
- once sync.Once
- result interface{}
- }
-
entryIface, ok := c.m.Load(key)
if !ok {
- entryIface, _ = c.m.LoadOrStore(key, new(entry))
+ entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry))
}
- e := entryIface.(*entry)
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.mu.Lock()
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.result = f()
+ atomic.StoreUint32(&e.done, 1)
+ }
+ e.mu.Unlock()
+ }
+ return e.result
+}
- e.once.Do(func() {
- e.result = f()
- })
+// Get returns the cached result associated with key.
+// It returns nil if there is no such result.
+// If the result for key is being computed, Get does not wait for the computation to finish.
+func (c *Cache) Get(key interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ return nil
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ return nil
+ }
return e.result
}
diff --git a/vendor/cmd/go/internal/vgo/build.go b/vendor/cmd/go/internal/vgo/build.go
index 8528090..b06213f 100644
--- a/vendor/cmd/go/internal/vgo/build.go
+++ b/vendor/cmd/go/internal/vgo/build.go
@@ -53,7 +53,7 @@
return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]})
}
- for _, m := range buildList {
+ for _, m := range BuildList() {
if m.Path == path {
return moduleInfo(m)
}
diff --git a/vendor/cmd/go/internal/vgo/get.go b/vendor/cmd/go/internal/vgo/get.go
index 45bf1ce..fcc76d4 100644
--- a/vendor/cmd/go/internal/vgo/get.go
+++ b/vendor/cmd/go/internal/vgo/get.go
@@ -114,7 +114,7 @@
// Upgrade.
var err error
- buildList, err = mvs.Upgrade(Target, newReqs(), upgrade...)
+ buildList, err = mvs.Upgrade(Target, newReqs(buildList), upgrade...)
if err != nil {
base.Fatalf("vgo get: %v", err)
}
@@ -133,7 +133,7 @@
}
if len(downgrade) > 0 {
- buildList, err = mvs.Downgrade(Target, newReqs(buildList[1:]...), downgrade...)
+ buildList, err = mvs.Downgrade(Target, newReqs(buildList, buildList[1:]...), downgrade...)
if err != nil {
base.Fatalf("vgo get: %v", err)
}
@@ -160,7 +160,7 @@
if *getD {
// Download all needed code as side-effect.
- ImportPaths([]string{"ALL"})
+ LoadALL()
}
if *getM {
diff --git a/vendor/cmd/go/internal/vgo/init.go b/vendor/cmd/go/internal/vgo/init.go
index 5352fd3..4904001 100644
--- a/vendor/cmd/go/internal/vgo/init.go
+++ b/vendor/cmd/go/internal/vgo/init.go
@@ -48,11 +48,6 @@
)
-// TargetPackages returns the list of packages in the target (top-level) module.
-func TargetPackages() []string {
- return matchPackages("ALL", []module.Version{Target})
-}
-
// ModFile returns the parsed go.mod file.
//
// Note that after calling ImportPaths or LoadBuildList,
@@ -154,7 +149,7 @@
load.VgoImportPaths = ImportPaths
load.VgoPackageBuildInfo = PackageBuildInfo
load.VgoModInfoProg = ModInfoProg
- load.VgoAddImports = AddImports
+ load.VgoImportFromFiles = ImportFromFiles
search.SetModRoot(ModRoot)
}
@@ -423,7 +418,7 @@
modfetch.WriteGoSum()
if buildList != nil {
- min, err := mvs.Req(Target, buildList, newReqs())
+ min, err := mvs.Req(Target, buildList, newReqs(buildList))
if err != nil {
base.Fatalf("vgo: %v", err)
}
diff --git a/vendor/cmd/go/internal/vgo/load.go b/vendor/cmd/go/internal/vgo/load.go
index 62d24bf..342fda8 100644
--- a/vendor/cmd/go/internal/vgo/load.go
+++ b/vendor/cmd/go/internal/vgo/load.go
@@ -6,6 +6,7 @@
import (
"bytes"
+ "errors"
"fmt"
"go/build"
"io/ioutil"
@@ -13,6 +14,7 @@
"path/filepath"
"sort"
"strings"
+ "sync"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
@@ -26,37 +28,145 @@
"cmd/go/internal/semver"
)
-type importLevel int
+// buildList is the list of modules to use for building packages.
+// It is initialized by calling ImportPaths, ImportFromFiles,
+// LoadALL, or LoadBuildList, each of which uses loaded.load.
+//
+// Ideally, exactly ONE of those functions would be called,
+// and exactly once. Most of the time, that's true.
+// During "go get" it may not be. TODO(rsc): Figure out if
+// that restriction can be established, or else document why not.
+//
+var buildList []module.Version
-const (
- levelNone importLevel = 0
- levelBuild importLevel = 1
- levelTest importLevel = 2
- levelTestRecursive importLevel = 3
-)
+// loaded is the most recently-used package loader.
+// It holds details about individual packages.
+//
+// Note that loaded.buildList is only valid during a load operation;
+// afterward, it is copied back into the global buildList,
+// which should be used instead.
+var loaded *loader
-var (
- buildList []module.Version
- tags map[string]bool
- importmap map[string]string
- pkgdir map[string]string
- pkgmod map[string]module.Version
-)
+// ImportPaths returns the set of packages matching the args (patterns),
+// adding modules to the build list as needed to satisfy new imports.
+func ImportPaths(args []string) []string {
+ if Init(); !Enabled() {
+ return search.ImportPaths(args)
+ }
+ InitMod()
-func AddImports(gofiles []string) {
+ cleaned := search.CleanImportPaths(args)
+ loaded = newLoader()
+ var paths []string
+ loaded.load(func() []string {
+ var roots []string
+ paths = nil
+ for _, pkg := range cleaned {
+ switch {
+ case build.IsLocalImport(pkg):
+ list := []string{pkg}
+ if strings.Contains(pkg, "...") {
+ // TODO: Where is the go.mod cutoff?
+ list = warnPattern(pkg, search.AllPackagesInFS(pkg))
+ }
+ for _, pkg := range list {
+ dir := filepath.Join(cwd, pkg)
+ if dir == ModRoot {
+ pkg = Target.Path
+ } else if strings.HasPrefix(dir, ModRoot+string(filepath.Separator)) {
+ suffix := filepath.ToSlash(dir[len(ModRoot):])
+ if strings.HasPrefix(suffix, "/vendor/") {
+ // TODO getmode vendor check
+ pkg = strings.TrimPrefix(suffix, "/vendor/")
+ } else {
+ pkg = Target.Path + suffix
+ }
+ } else {
+ base.Errorf("vgo: package %s outside module root", pkg)
+ continue
+ }
+ roots = append(roots, pkg)
+ paths = append(paths, pkg)
+ }
+
+ case pkg == "all":
+ if loaded.testRoots {
+ loaded.testAll = true
+ }
+ // TODO: Don't print warnings multiple times.
+ roots = append(roots, warnPattern("all", matchPackages("...", loaded.tags, []module.Version{Target}))...)
+ paths = append(paths, "all") // will expand after load completes
+
+ case search.IsMetaPackage(pkg): // std, cmd
+ fmt.Fprintf(os.Stderr, "vgo: warning: %q matches no packages when using modules\n", pkg)
+
+ case strings.Contains(pkg, "..."):
+ // TODO: Don't we need to reevaluate this one last time once the build list stops changing?
+ list := warnPattern(pkg, matchPackages(pkg, loaded.tags, loaded.buildList))
+ roots = append(roots, list...)
+ paths = append(paths, list...)
+
+ default:
+ roots = append(roots, pkg)
+ paths = append(paths, pkg)
+ }
+ }
+ return roots
+ })
+ WriteGoMod()
+
+ // Process paths to produce final paths list.
+ // Remove duplicates and expand "all".
+ have := make(map[string]bool)
+ var final []string
+ for _, path := range paths {
+ if have[path] {
+ continue
+ }
+ have[path] = true
+ if path == "all" {
+ for _, pkg := range loaded.pkgs {
+ if !have[pkg.path] {
+ have[pkg.path] = true
+ final = append(final, pkg.path)
+ }
+ }
+ continue
+ }
+ final = append(final, path)
+ }
+ return final
+}
+
+// warnPattern returns list, the result of matching pattern,
+// but if list is empty then first it prints a warning about
+// the pattern not matching any packages.
+func warnPattern(pattern string, list []string) []string {
+ if len(list) == 0 {
+ fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
+ }
+ return list
+}
+
+// ImportFromFiles adds modules to the build list as needed
+// to satisfy the imports in the named Go source files.
+func ImportFromFiles(gofiles []string) {
if Init(); !Enabled() {
return
}
InitMod()
- imports, testImports, err := imports.ScanFiles(gofiles, tags)
+ imports, testImports, err := imports.ScanFiles(gofiles, imports.Tags())
if err != nil {
base.Fatalf("vgo: %v", err)
}
- iterate(func(ld *loader) {
- ld.importList(imports, levelBuild)
- ld.importList(testImports, levelBuild)
+ loaded = newLoader()
+ loaded.load(func() []string {
+ var roots []string
+ roots = append(roots, imports...)
+ roots = append(roots, testImports...)
+ return roots
})
WriteGoMod()
}
@@ -71,11 +181,48 @@
base.Fatalf("vgo: LoadBuildList called but vgo not enabled")
}
InitMod()
- iterate(func(*loader) {})
+ loaded = newLoader()
+ loaded.load(func() []string { return nil })
WriteGoMod()
return buildList
}
+// LoadALL returns the set of all packages in the current module
+// and their dependencies in any other modules, without filtering
+// due to build tags, except "+build ignore".
+// It adds modules to the build list as needed to satisfy new imports.
+// This set is useful for identifying which packages to include in a vendor directory
+// or deciding whether a particular import appears anywhere in a module.
+func LoadALL() []string {
+ if Init(); !Enabled() {
+ panic("vgo: misuse of LoadALL")
+ }
+ InitMod()
+
+ loaded = newLoader()
+ loaded.tags = anyTags
+ loaded.testAll = true
+ all := TargetPackages()
+ loaded.load(func() []string { return all })
+ WriteGoMod()
+
+ var paths []string
+ for _, pkg := range loaded.pkgs {
+ paths = append(paths, pkg.path)
+ }
+ return paths
+}
+
+// anyTags is a special tags map that satisfies nearly all build tag expressions.
+// Only "ignore" and malformed build tag requirements are considered false.
+var anyTags = map[string]bool{"*": true}
+
+// TargetPackages returns the list of packages in the target (top-level) module,
+// under all build tag settings.
+func TargetPackages() []string {
+ return matchPackages("...", anyTags, []module.Version{Target})
+}
+
// BuildList returns the module build list,
// typically constructed by a previous call to
// LoadBuildList or ImportPaths.
@@ -94,65 +241,34 @@
// If the given import path does not appear in the source code
// for the packages that have been loaded, ImportMap returns the empty string.
func ImportMap(path string) string {
- return importmap[path]
+ pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
+ if !ok {
+ return ""
+ }
+ return pkg.path
}
// PackageDir returns the directory containing the source code
// for the package named by the import path.
func PackageDir(path string) string {
- return pkgdir[path]
+ pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
+ if !ok {
+ return ""
+ }
+ return pkg.dir
}
// PackageModule returns the module providing the package named by the import path.
func PackageModule(path string) module.Version {
- return pkgmod[path]
-}
-
-func ImportPaths(args []string) []string {
- if Init(); !Enabled() {
- return search.ImportPaths(args)
+ pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
+ if !ok {
+ return module.Version{}
}
- InitMod()
-
- paths := importPaths(args)
- WriteGoMod()
- return paths
-}
-
-func importPaths(args []string) []string {
- level := levelBuild
- switch cfg.CmdName {
- case "test", "vet":
- level = levelTest
- }
- isALL := len(args) == 1 && args[0] == "ALL"
- cleaned := search.CleanImportPaths(args)
- iterate(func(ld *loader) {
- if isALL {
- ld.tags = map[string]bool{"*": true}
- }
- args = expandImportPaths(cleaned)
- for i, pkg := range args {
- if pkg == "." || pkg == ".." || strings.HasPrefix(pkg, "./") || strings.HasPrefix(pkg, "../") {
- dir := filepath.Join(cwd, pkg)
- if dir == ModRoot {
- pkg = Target.Path
- } else if strings.HasPrefix(dir, ModRoot+string(filepath.Separator)) {
- pkg = Target.Path + filepath.ToSlash(dir[len(ModRoot):])
- } else {
- base.Errorf("vgo: package %s outside module root", pkg)
- continue
- }
- args[i] = pkg
- }
- ld.importPkg(pkg, level)
- }
- })
- return args
+ return pkg.mod
}
func Lookup(parentPath, path string) (dir, realPath string, err error) {
- realPath = importmap[path]
+ realPath = ImportMap(path)
if realPath == "" {
if isStandardImportPath(path) {
dir := filepath.Join(cfg.GOROOT, "src", path)
@@ -162,192 +278,426 @@
}
return "", "", fmt.Errorf("no such package in module")
}
- return pkgdir[realPath], realPath, nil
+ return PackageDir(realPath), realPath, nil
}
-func iterate(doImports func(*loader)) {
+// A loader manages the process of loading information about
+// the required packages for a particular build,
+// checking that the packages are available in the module set,
+// and updating the module set if needed.
+// Loading is an iterative process: try to load all the needed packages,
+// but if imports are missing, try to resolve those imports, and repeat.
+//
+// Although most of the loading state is maintained in the loader struct,
+// one key piece - the build list - is a global, so that it can be modified
+// separate from the loading operation, such as during "go get"
+// upgrades/downgrades or in "go mod" operations.
+// TODO(rsc): It might be nice to make the loader take and return
+// a buildList rather than hard-coding use of the global.
+type loader struct {
+ tags map[string]bool // tags for scanDir
+ testRoots bool // include tests for roots
+ testAll bool // include tests for all packages
+
+ // missingMu protects found, but also buildList, modFile
+ missingMu sync.Mutex
+ found map[string]bool
+
+ // updated on each iteration
+ buildList []module.Version
+
+ // reset on each iteration
+ roots []*loadPkg
+ pkgs []*loadPkg
+ work *par.Work // current work queue
+ pkgCache *par.Cache // map from string to *loadPkg
+ missing *par.Work // missing work queue
+}
+
+func newLoader() *loader {
+ ld := new(loader)
+ ld.tags = imports.Tags()
+ ld.found = make(map[string]bool)
+
+ switch cfg.CmdName {
+ case "test", "vet":
+ ld.testRoots = true
+ }
+ return ld
+}
+
+func (ld *loader) reset() {
+ ld.roots = nil
+ ld.pkgs = nil
+ ld.work = new(par.Work)
+ ld.pkgCache = new(par.Cache)
+ ld.missing = nil
+}
+
+// A loadPkg records information about a single loaded package.
+type loadPkg struct {
+ path string // import path
+ mod module.Version // module providing package
+ dir string // directory containing source code
+ imports []*loadPkg // packages imported by this one
+ err error // error loading package
+ stack *loadPkg // package importing this one in minimal import stack for this pkg
+ test *loadPkg // package with test imports, if we need test
+ testOf *loadPkg
+ testImports []string // test-only imports, saved for use by pkg.test.
+}
+
+var errMissing = errors.New("cannot find package")
+
+// load attempts to load the build graph needed to process a set of root packages.
+// The set of root packages is defined by the addRoots function,
+// which must call add(path) with the import path of each root package.
+func (ld *loader) load(roots func() []string) {
var err error
mvsOp := mvs.BuildList
if *getU {
mvsOp = mvs.UpgradeAll
}
- buildList, err = mvsOp(Target, newReqs())
+ ld.buildList = buildList
+ ld.buildList, err = mvsOp(Target, newReqs(ld.buildList))
if err != nil {
base.Fatalf("vgo: %v", err)
}
- var ld *loader
for {
- ld = newLoader()
- doImports(ld)
- if len(ld.missing) == 0 {
+ ld.reset()
+ if roots != nil {
+ // Note: the returned roots can change on each iteration,
+ // since the expansion of package patterns depends on the
+ // build list we're using.
+ for _, path := range roots() {
+ ld.work.Add(ld.pkg(path, true))
+ }
+ }
+ ld.work.Do(10, ld.doPkg)
+ ld.buildStacks()
+ for _, pkg := range ld.pkgs {
+ if pkg.err == errMissing {
+ if ld.missing == nil {
+ ld.missing = new(par.Work)
+ }
+ ld.missing.Add(pkg)
+ } else if pkg.err != nil {
+ base.Errorf("vgo: %s: %s", pkg.stackText(), pkg.err)
+ }
+ }
+ if ld.missing == nil {
break
}
- for _, m := range ld.missing {
- findMissing(m)
- }
+ ld.missing.Do(10, ld.findMissing)
base.ExitIfErrors()
- buildList, err = mvsOp(Target, newReqs())
+
+ ld.buildList, err = mvsOp(Target, newReqs(ld.buildList))
if err != nil {
base.Fatalf("vgo: %v", err)
}
}
base.ExitIfErrors()
- importmap = ld.importmap
- pkgdir = ld.pkgdir
- pkgmod = ld.pkgmod
+ buildList = ld.buildList
+ ld.buildList = nil // catch accidental use
}
-type loader struct {
- imported map[string]importLevel
- importmap map[string]string
- pkgdir map[string]string
- pkgmod map[string]module.Version
- tags map[string]bool
- missing []missing
- imports []string
- stack []string
+// pkg returns the *loadPkg for path, creating and queuing it if needed.
+// If the package should be tested, its test is created but not queued
+// (the test is queued after processing pkg).
+// If isRoot is true, the pkg is being queued as one of the roots of the work graph.
+func (ld *loader) pkg(path string, isRoot bool) *loadPkg {
+ return ld.pkgCache.Do(path, func() interface{} {
+ pkg := &loadPkg{
+ path: path,
+ }
+ if ld.testRoots && isRoot || ld.testAll {
+ test := &loadPkg{
+ path: path,
+ testOf: pkg,
+ }
+ pkg.test = test
+ }
+ if isRoot {
+ ld.roots = append(ld.roots, pkg)
+ }
+ ld.work.Add(pkg)
+ return pkg
+ }).(*loadPkg)
}
-type missing struct {
- path string
- stack string
-}
-
-func newLoader() *loader {
- ld := &loader{
- imported: make(map[string]importLevel),
- importmap: make(map[string]string),
- pkgdir: make(map[string]string),
- pkgmod: make(map[string]module.Version),
- tags: imports.Tags(),
- }
- ld.imported["C"] = 100
- return ld
-}
-
-func (ld *loader) stackText() string {
- var buf bytes.Buffer
- for _, p := range ld.stack[:len(ld.stack)-1] {
- fmt.Fprintf(&buf, "import %q ->\n\t", p)
- }
- fmt.Fprintf(&buf, "import %q", ld.stack[len(ld.stack)-1])
- return buf.String()
-}
-
-func (ld *loader) importList(pkgs []string, level importLevel) {
- for _, pkg := range pkgs {
- ld.importPkg(pkg, level)
- }
-}
-
-func (ld *loader) importPkg(path string, level importLevel) {
- if ld.imported[path] >= level {
- return
- }
-
- ld.stack = append(ld.stack, path)
- defer func() {
- ld.stack = ld.stack[:len(ld.stack)-1]
- }()
-
- // Any rewritings go here.
- realPath := path
-
- ld.imported[path] = level
- ld.importmap[path] = realPath
- if realPath != path && ld.imported[realPath] >= level {
- // Already handled.
- return
- }
-
- dir := ld.importDir(realPath)
- if dir == "" {
- return
- }
-
- ld.pkgdir[realPath] = dir
-
- imports, testImports, err := scanDir(dir, ld.tags)
- if err != nil {
- if strings.HasPrefix(err.Error(), "no Go ") {
- // Don't print about directories with no Go source files.
- // Let the eventual real package load do that.
+// doPkg processes a package on the work queue.
+func (ld *loader) doPkg(item interface{}) {
+ // TODO: what about replacements?
+ pkg := item.(*loadPkg)
+ var imports []string
+ if pkg.testOf != nil {
+ pkg.dir = pkg.testOf.dir
+ pkg.mod = pkg.testOf.mod
+ imports = pkg.testOf.testImports
+ } else {
+ pkg.dir, pkg.mod, pkg.err = ld.findDir(pkg.path)
+ if pkg.dir == "" {
return
}
- base.Errorf("vgo: %s [%s]: %v", ld.stackText(), dir, err)
- return
- }
- nextLevel := level
- if level == levelTest {
- nextLevel = levelBuild
- }
- for _, pkg := range imports {
- ld.importPkg(pkg, nextLevel)
- }
- if level >= levelTest {
- for _, pkg := range testImports {
- ld.importPkg(pkg, nextLevel)
+ var testImports []string
+ var err error
+ imports, testImports, err = scanDir(pkg.dir, ld.tags)
+ if err != nil {
+ if strings.HasPrefix(err.Error(), "no Go ") {
+ // Don't print about directories with no Go source files.
+ // Let the eventual real package load do that.
+ return
+ }
+ pkg.err = err
+ return
}
+ if pkg.test != nil {
+ pkg.testImports = testImports
+ }
+ }
+
+ for _, path := range imports {
+ pkg.imports = append(pkg.imports, ld.pkg(path, false))
+ }
+
+ // Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test.
+ // TODO: All that's left is creating new imports. Why not just do it now?
+ if pkg.test != nil {
+ ld.work.Add(pkg.test)
}
}
-func (ld *loader) importDir(path string) string {
- if importPathInModule(path, Target.Path) {
- dir := ModRoot
- if len(path) > len(Target.Path) {
- dir = filepath.Join(dir, path[len(Target.Path)+1:])
- }
- ld.pkgmod[path] = Target
- return dir
- }
+// importPathInModule reports whether, syntactically,
+// a package with the given import path could be supplied
+// by a module with the given module path (mpath).
+func importPathInModule(path, mpath string) bool {
+ return mpath == path ||
+ len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath
+}
+// findDir finds the directory holding source code for the given import path.
+// It returns the directory, the module containing the directory,
+// and any error encountered.
+// It is possible to return successfully (err == nil) with an empty directory,
+// for built-in packages like "unsafe" and "C".
+// It is also possible to return successfully with a zero module.Version,
+// for packages in the standard library or when using vendored code.
+func (ld *loader) findDir(path string) (dir string, mod module.Version, err error) {
+ // Is the package in the standard library?
if search.IsStandardImportPath(path) {
+ if path == "C" || path == "unsafe" {
+ // There's no directory for import "C" or import "unsafe".
+ return "", module.Version{}, nil
+ }
if strings.HasPrefix(path, "golang_org/") {
- return filepath.Join(cfg.GOROOT, "src/vendor", path)
+ return filepath.Join(cfg.GOROOT, "src/vendor", path), module.Version{}, nil
}
dir := filepath.Join(cfg.GOROOT, "src", path)
if _, err := os.Stat(dir); err == nil {
- return dir
+ return dir, module.Version{}, nil
}
}
+ // Is the package in the main module?
+ // Note that having the main module path as a prefix
+ // does not guarantee that the package is in the
+ // main module. It might still be supplied by some
+ // other module. For example, this might be
+ // module x/y, and we might be looking for x/y/v2/z.
+ // or maybe x/y/z/w in separate module x/y/z.
+ var mainDir string
+ if importPathInModule(path, Target.Path) {
+ mainDir = ModRoot
+ if len(path) > len(Target.Path) {
+ mainDir = filepath.Join(ModRoot, path[len(Target.Path)+1:])
+ }
+ if _, err := os.Stat(mainDir); err == nil {
+ return mainDir, Target, nil
+ }
+ }
+
+ // With -getmode=vendor, we expect everything else to be in vendor.
if cfg.BuildGetmode == "vendor" {
// Using -getmode=vendor, everything the module needs
// (beyond the current module and standard library)
// must be in the module's vendor directory.
- return filepath.Join(ModRoot, "vendor", path)
+ // If the package exists in vendor, use it.
+ // If the package is not covered by the main module (mainDir == ""), use vendor.
+ // Otherwise, if the package could be in either place but is in neither, report the main module.
+ vendorDir := filepath.Join(ModRoot, "vendor", path)
+ if _, err := os.Stat(vendorDir); err == nil || mainDir == "" {
+ // TODO(rsc): We could look up the module information from vendor/modules.txt.
+ return vendorDir, module.Version{}, nil
+ }
+ return mainDir, Target, nil
}
+ // Scan all the possible modules that might contain this package,
+ // and complain if there are multiple choices. This correctly handles
+ // module boundaries that change over time, detecting mismatched
+ // module version pairings.
+ // (See comment about module paths in modfetch/repo.go.)
var mod1 module.Version
var dir1 string
- for _, mod := range buildList {
+ for _, mod := range ld.buildList {
if !importPathInModule(path, mod.Path) {
continue
}
dir, err := fetch(mod)
if err != nil {
- base.Errorf("vgo: %s: %v", ld.stackText(), err)
- return ""
+ return "", module.Version{}, err
}
if len(path) > len(mod.Path) {
dir = filepath.Join(dir, path[len(mod.Path)+1:])
}
if dir1 != "" {
- base.Errorf("vgo: %s: found in both %v %v and %v %v", ld.stackText(),
- mod1.Path, mod1.Version, mod.Path, mod.Version)
- return ""
+ return "", module.Version{}, fmt.Errorf("found in both %v@%v and %v@%v", mod1.Path, mod1.Version, mod.Path, mod.Version)
}
dir1 = dir
mod1 = mod
}
if dir1 != "" {
- ld.pkgmod[path] = mod1
- return dir1
+ return dir1, mod1, nil
}
- ld.missing = append(ld.missing, missing{path, ld.stackText()})
- return ""
+ return "", module.Version{}, errMissing
+}
+
+func (ld *loader) findMissing(item interface{}) {
+ pkg := item.(*loadPkg)
+ path := pkg.path
+ if build.IsLocalImport(path) {
+ base.Errorf("vgo: relative import is not supported: %s", path)
+ }
+
+ // TODO: This is wrong (if path = foo/v2/bar and m.Path is foo,
+ // maybe we should fall through to the loop at the bottom and check foo/v2).
+ ld.missingMu.Lock()
+ for _, m := range ld.buildList {
+ if importPathInModule(path, m.Path) {
+ ld.missingMu.Unlock()
+ return
+ }
+ }
+ ld.missingMu.Unlock()
+
+ fmt.Fprintf(os.Stderr, "resolving import %q\n", path)
+ repo, info, err := modfetch.Import(path, allowed)
+ if err != nil {
+ base.Errorf("vgo: %s: %v", pkg.stackText(), err)
+ return
+ }
+
+ root := repo.ModulePath()
+
+ ld.missingMu.Lock()
+ defer ld.missingMu.Unlock()
+
+ // Double-check before adding repo twice.
+ for _, m := range ld.buildList {
+ if importPathInModule(path, m.Path) {
+ return
+ }
+ }
+
+ fmt.Fprintf(os.Stderr, "vgo: finding %s (latest)\n", root)
+
+ if ld.found[path] {
+ base.Fatalf("internal error: findMissing loop on %s", path)
+ }
+ ld.found[path] = true
+ fmt.Fprintf(os.Stderr, "vgo: adding %s %s\n", root, info.Version)
+ ld.buildList = append(ld.buildList, module.Version{Path: root, Version: info.Version})
+ modFile.AddRequire(root, info.Version)
+}
+
+// scanDir is like imports.ScanDir but elides known magic imports from the list,
+// so that vgo does not go looking for packages that don't really exist.
+//
+// The only known magic imports are appengine and appengine/*.
+// These are so old that they predate "go get" and did not use URL-like paths.
+// Most code today now uses google.golang.org/appengine instead,
+// but not all code has been so updated. When we mostly ignore build tags
+// during "vgo vendor", we look into "// +build appengine" files and
+// may see these legacy imports. We drop them so that the module
+// search does not look for modules to try to satisfy them.
+func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) {
+ imports_, testImports, err = imports.ScanDir(dir, tags)
+
+ filter := func(x []string) []string {
+ w := 0
+ for _, pkg := range x {
+ if pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") &&
+ pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") {
+ x[w] = pkg
+ w++
+ }
+ }
+ return x[:w]
+ }
+
+ return filter(imports_), filter(testImports), err
+}
+
+// buildStacks computes minimal import stacks for each package,
+// for use in error messages. When it completes, packages that
+// are part of the original root set have pkg.stack == nil,
+// and other packages have pkg.stack pointing at the next
+// package up the import stack in their minimal chain.
+// As a side effect, buildStacks also constructs ld.pkgs,
+// the list of all packages loaded.
+func (ld *loader) buildStacks() {
+ if len(ld.pkgs) > 0 {
+ panic("buildStacks")
+ }
+ for _, pkg := range ld.roots {
+ pkg.stack = pkg // sentinel to avoid processing in next loop
+ ld.pkgs = append(ld.pkgs, pkg)
+ }
+ for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop
+ pkg := ld.pkgs[i]
+ for _, next := range pkg.imports {
+ if next.stack == nil {
+ next.stack = pkg
+ ld.pkgs = append(ld.pkgs, next)
+ }
+ }
+ if next := pkg.test; next != nil && next.stack == nil {
+ next.stack = pkg
+ ld.pkgs = append(ld.pkgs, next)
+ }
+ }
+ for _, pkg := range ld.roots {
+ pkg.stack = nil
+ }
+}
+
+// stackText builds the import stack text to use when
+// reporting an error in pkg. It has the general form
+//
+// import root ->
+// import other ->
+// import other2 ->
+// import pkg
+//
+func (pkg *loadPkg) stackText() string {
+ var stack []*loadPkg
+ for p := pkg.stack; p != nil; p = p.stack {
+ stack = append(stack, p)
+ }
+
+ var buf bytes.Buffer
+ for i := len(stack) - 1; i >= 0; i-- {
+ p := stack[i]
+ if p.testOf != nil {
+ fmt.Fprintf(&buf, "test ->\n\t")
+ } else {
+ fmt.Fprintf(&buf, "import %q ->\n\t", p.path)
+ }
+ }
+ fmt.Fprintf(&buf, "import %q", pkg.path)
+ return buf.String()
}
// Replacement returns the replacement for mod, if any, from go.mod.
@@ -366,58 +716,25 @@
return found.New
}
-func importPathInModule(path, mpath string) bool {
- return mpath == path ||
- len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath
-}
-
-var found = make(map[string]bool)
-
-func findMissing(m missing) {
- for _, mod := range buildList {
- if importPathInModule(m.path, mod.Path) {
- // Leave for ordinary build to complain about the missing import.
- return
- }
- }
- if build.IsLocalImport(m.path) {
- base.Errorf("vgo: relative import is not supported: %s", m.path)
- return
- }
- fmt.Fprintf(os.Stderr, "vgo: resolving import %q\n", m.path)
- repo, info, err := modfetch.Import(m.path, allowed)
- if err != nil {
- base.Errorf("vgo: %s: %v", m.stack, err)
- return
- }
- root := repo.ModulePath()
- fmt.Fprintf(os.Stderr, "vgo: finding %s (latest)\n", root)
- if found[root] {
- base.Fatalf("internal error: findmissing loop on %s", root)
- }
- found[root] = true
- fmt.Fprintf(os.Stderr, "vgo: adding %s %s\n", root, info.Version)
- buildList = append(buildList, module.Version{Path: root, Version: info.Version})
- modFile.AddRequire(root, info.Version)
-}
-
-// mvsReqs implements mvs.Reqs for vgo's semantic versions, with any exclusions
-// or replacements applied internally.
+// mvsReqs implements mvs.Reqs for vgo's semantic versions,
+// with any exclusions or replacements applied internally.
type mvsReqs struct {
- extra []module.Version
- cache par.Cache
+ buildList []module.Version
+ extra []module.Version
+ cache par.Cache
}
-func newReqs(extra ...module.Version) *mvsReqs {
+func newReqs(buildList []module.Version, extra ...module.Version) *mvsReqs {
r := &mvsReqs{
- extra: extra,
+ buildList: buildList,
+ extra: extra,
}
return r
}
// Reqs returns the module requirement graph.
func Reqs() mvs.Reqs {
- return newReqs()
+ return newReqs(buildList)
}
func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) {
@@ -454,8 +771,8 @@
func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) {
if mod == Target {
var list []module.Version
- if buildList != nil {
- list = append(list, buildList[1:]...)
+ if r.buildList != nil {
+ list = append(list, r.buildList[1:]...)
return list, nil
}
for _, r := range modFile.Require {
@@ -611,34 +928,6 @@
return module.Version{Path: m.Path, Version: "none"}, nil
}
-// scanDir is like imports.ScanDir but elides known magic imports from the list,
-// so that vgo does not go looking for packages that don't really exist.
-//
-// The only known magic imports are appengine and appengine/*.
-// These are so old that they predate "go get" and did not use URL-like paths.
-// Most code today now uses google.golang.org/appengine instead,
-// but not all code has been so updated. When we mostly ignore build tags
-// during "vgo vendor", we look into "// +build appengine" files and
-// may see these legacy imports. We drop them so that the module
-// search does not look for modules to try to satisfy them.
-func scanDir(path string, tags map[string]bool) (imports_, testImports []string, err error) {
- imports_, testImports, err = imports.ScanDir(path, tags)
-
- filter := func(x []string) []string {
- w := 0
- for _, pkg := range x {
- if pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") &&
- pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") {
- x[w] = pkg
- w++
- }
- }
- return x
- }
-
- return filter(imports_), filter(testImports), err
-}
-
func fetch(mod module.Version) (dir string, err error) {
if r := Replacement(mod); r.Path != "" {
if r.Version == "" {
diff --git a/vendor/cmd/go/internal/vgo/search.go b/vendor/cmd/go/internal/vgo/search.go
index fb2bac4..9e2bc51 100644
--- a/vendor/cmd/go/internal/vgo/search.go
+++ b/vendor/cmd/go/internal/vgo/search.go
@@ -6,10 +6,8 @@
import (
"fmt"
- "go/build"
"os"
"path/filepath"
- "sort"
"strings"
"cmd/go/internal/base"
@@ -19,65 +17,12 @@
"cmd/go/internal/search"
)
-func expandImportPaths(args []string) []string {
- var out []string
- for _, a := range args {
- // TODO(rsc): Move a == "ALL" test into search.IsMetaPackage
- // once we officially lock in all the module work (tentatively, Go 1.12).
- if search.IsMetaPackage(a) || a == "ALL" {
- switch a {
- default:
- fmt.Fprintf(os.Stderr, "vgo: warning: %q matches no packages when using modules\n", a)
- case "all", "ALL":
- out = append(out, AllPackages(a)...)
- }
- continue
- }
- if strings.Contains(a, "...") {
- if build.IsLocalImport(a) {
- out = append(out, search.AllPackagesInFS(a)...)
- } else {
- out = append(out, AllPackages(a)...)
- }
- continue
- }
- out = append(out, a)
- }
- return out
-}
-
-// AllPackages returns all the packages that can be found
-// under the $GOPATH directories and $GOROOT matching pattern.
-// The pattern is either "all" (all packages), "std" (standard packages),
-// "cmd" (standard commands), or a path including "...".
-func AllPackages(pattern string) []string {
- pkgs := MatchPackages(pattern)
- if len(pkgs) == 0 {
- fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
- }
- return pkgs
-}
-
-// MatchPackages returns a list of package paths matching pattern
-// (see go help packages for pattern syntax).
-func MatchPackages(pattern string) []string {
- if pattern == "std" || pattern == "cmd" {
- return nil
- }
- if pattern == "all" {
- return MatchAll()
- }
- if pattern == "ALL" {
- return MatchALL()
- }
-
- return matchPackages(pattern, buildList)
-}
-
-func matchPackages(pattern string, buildList []module.Version) []string {
+// matchPackages returns a list of packages in the list of modules
+// matching the pattern. Package loading assumes the given set of tags.
+func matchPackages(pattern string, tags map[string]bool, modules []module.Version) []string {
match := func(string) bool { return true }
treeCanMatch := func(string) bool { return true }
- if !search.IsMetaPackage(pattern) && pattern != "ALL" {
+ if !search.IsMetaPackage(pattern) {
match = search.MatchPattern(pattern)
treeCanMatch = search.TreeCanMatchPattern(pattern)
}
@@ -90,7 +35,7 @@
}
var pkgs []string
- for _, mod := range buildList {
+ for _, mod := range modules {
if !treeCanMatch(mod.Path) {
continue
}
@@ -145,7 +90,7 @@
if !have[name] {
have[name] = true
if match(name) {
- if _, _, err := scanDir(path, imports.Tags()); err != imports.ErrNoGo {
+ if _, _, err := scanDir(path, tags); err != imports.ErrNoGo {
pkgs = append(pkgs, name)
}
}
@@ -159,38 +104,3 @@
}
return pkgs
}
-
-// MatchAll returns a list of the packages matching the pattern "all".
-// We redefine "all" to mean start with the packages in the current module
-// and then follow imports into other modules to add packages imported
-// (directly or indirectly) as part of builds in this module.
-// It does not include packages in other modules that are not needed
-// by builds of this module.
-func MatchAll() []string {
- return matchAll(imports.Tags())
-}
-
-// MatchALL returns a list of the packages matching the pattern "ALL".
-// The pattern "ALL" is like "all" but looks at all source files,
-// even ones that would be ignored by current build tag settings.
-// That's useful for identifying which packages to include in a vendor directory.
-func MatchALL() []string {
- return matchAll(map[string]bool{"*": true})
-}
-
-// matchAll is the common implementation of MatchAll and MatchALL,
-// which differ only in the set of tags to apply to select files.
-func matchAll(tags map[string]bool) []string {
- local := matchPackages("all", buildList[:1])
- ld := newLoader()
- ld.tags = tags
- ld.importList(local, levelTestRecursive)
- var all []string
- for _, pkg := range ld.importmap {
- if !isStandardImportPath(pkg) {
- all = append(all, pkg)
- }
- }
- sort.Strings(all)
- return all
-}
diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go
index ef74dce..e38ee24 100644
--- a/vendor/cmd/go/vgo_test.go
+++ b/vendor/cmd/go/vgo_test.go
@@ -568,10 +568,9 @@
if !testing.Short() {
tg.run("-vgo", "build")
tg.run("-vgo", "build", "-getmode=vendor")
+ tg.cd(filepath.Join(wd, "testdata/vendormod/vendor"))
+ tg.run("-vgo", "test", "-getmode=vendor", "./...")
}
- //test testdata copy
- tg.cd(filepath.Join(wd, "testdata/vendormod/vendor"))
- tg.run("-vgo", "test", "./...")
}
func TestFillGoMod(t *testing.T) {
