cmd/go: parallelize matchPackages work in each module
In each module matchPackages looks in, when doing the walk, do the
scanDir call in a par.Queue so all the read work can be done in
parallel.
Change-Id: I27153dbb3a2ed417ca24972f47134e9e914a55d1
Reviewed-on: https://go-review.googlesource.com/c/go/+/404097
Reviewed-by: Michael Matloob <matloob@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go
index 60c6886..4b90392 100644
--- a/src/cmd/go/internal/modload/search.go
+++ b/src/cmd/go/internal/modload/search.go
@@ -12,12 +12,16 @@
"os"
"path"
"path/filepath"
+ "runtime"
+ "sort"
"strings"
+ "sync"
"cmd/go/internal/cfg"
"cmd/go/internal/fsys"
"cmd/go/internal/imports"
"cmd/go/internal/modindex"
+ "cmd/go/internal/par"
"cmd/go/internal/search"
"golang.org/x/mod/module"
@@ -43,9 +47,15 @@
treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
}
+ var mu sync.Mutex
have := map[string]bool{
"builtin": true, // ignore pseudo-package that exists only for documentation
}
+ addPkg := func(p string) {
+ mu.Lock()
+ m.Pkgs = append(m.Pkgs, p)
+ mu.Unlock()
+ }
if !cfg.BuildContext.CgoEnabled {
have["runtime/cgo"] = true // ignore during walk
}
@@ -56,6 +66,8 @@
pruneGoMod
)
+ q := par.NewQueue(runtime.GOMAXPROCS(0))
+
walkPkgs := func(root, importPathRoot string, prune pruning) {
root = filepath.Clean(root)
err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
@@ -110,9 +122,11 @@
if !have[name] {
have[name] = true
if isMatch(name) {
- if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
- m.Pkgs = append(m.Pkgs, name)
- }
+ q.Add(func() {
+ if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
+ addPkg(name)
+ }
+ })
}
}
@@ -126,6 +140,12 @@
}
}
+ // Wait for all in-flight operations to complete before returning.
+ defer func() {
+ <-q.Idle()
+ sort.Strings(m.Pkgs) // sort everything we added for determinism
+ }()
+
if filter == includeStd {
walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
if treeCanMatch("cmd") {
@@ -169,7 +189,7 @@
modPrefix = mod.Path
}
if mi, err := modindex.Get(root); err == nil {
- walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
+ walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
continue
} else if !errors.Is(err, modindex.ErrNotIndexed) {
m.AddError(err)
@@ -188,13 +208,7 @@
// walkFromIndex matches packages in a module using the module index. modroot
// is the module's root directory on disk, index is the ModuleIndex for the
// module, and importPathRoot is the module's path prefix.
-func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
- isMatch := func(string) bool { return true }
- treeCanMatch := func(string) bool { return true }
- if !m.IsMeta() {
- isMatch = search.MatchPattern(m.Pattern())
- treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
- }
+func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
loopPackages:
for _, reldir := range index.Packages() {
// Avoid .foo, _foo, and testdata subdirectory trees.
@@ -232,7 +246,7 @@
have[name] = true
if isMatch(name) {
if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
- m.Pkgs = append(m.Pkgs, name)
+ addPkg(name)
}
}
}
