cmd/go/internal/vgo: process packages for all build tags in vendor Until now, "vgo vendor" has only considered the packages needed for a build with the current GOOS/GOARCH and build tags. If a module has both linux-specific and windows-specific requirements, then "vgo vendor" on windows will omit the linux code, and vice versa, making it impossible to prepare a full vendor set. This CL introduces a new (for now undocumented) package pattern "ALL" that is like "all" but follows imports in files requiring any plausible build tag, producing a set of files that does not vary from system to system. The only +build lines not satisfied when computing "ALL" packages are those listing only the tag "ignore" or those listing only syntactically invalid build tags. For example, a file containing any of these lines not be processed: // +build ignore // +build ! // +build @#$%^&*() (The first two are commonly used in practice to exclude a file from processing by the go command.) This CL then changes the "vgo vendor" command to process "ALL" packages, not "all" packages. It also changes the vendor/vgo.list output file to list only modules that had files copied out of them and to indicate which packages came from which modules. Fixes golang/go#25376. Fixes golang/go#25624. Change-Id: I6756286fa6bb697819279ef67b4bea68bc0b7f53 Reviewed-on: https://go-review.googlesource.com/117258 Run-TryBot: Russ Cox <rsc@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Bryan C. Mills <bcmills@google.com>

diff --git a/vendor/cmd/go/internal/imports/build.go b/vendor/cmd/go/internal/imports/build.go
index bb40f2b..5597870 100644
--- a/vendor/cmd/go/internal/imports/build.go
+++ b/vendor/cmd/go/internal/imports/build.go

@@ -27,6 +27,12 @@
 //
 // marks the file as applicable only on Windows and Linux.
 //
+// If tags["*"] is true, then ShouldBuild will consider every
+// build tag except "ignore" to be both true and false for
+// the purpose of satisfying build tags, in order to estimate
+// (conservatively) whether a file could ever possibly be used
+// in any build.
+//
 func ShouldBuild(content []byte, tags map[string]bool) bool {
 	// Pass 1. Identify leading run of // comments and blank lines,
 	// which must be followed by a blank line.
@@ -87,8 +93,8 @@
 
 // matchTags reports whether the name is one of:
 //
-//	tag (if haveTags[tag] is true)
-//	!tag (if haveTags[tag] is false)
+//	tag (if tags[tag] is true)
+//	!tag (if tags[tag] is false)
 //	a comma-separated list of any of these
 //
 func matchTags(name string, tags map[string]bool) bool {
@@ -105,9 +111,13 @@
 		return false
 	}
 	if strings.HasPrefix(name, "!") { // negation
-		return len(name) > 1 && !matchTags(name[1:], tags)
+		return len(name) > 1 && matchTag(name[1:], tags, false)
 	}
+	return matchTag(name, tags, true)
+}
 
+// matchTag reports whether the tag name is valid and satisfied by tags[name]==want.
+func matchTag(name string, tags map[string]bool, want bool) bool {
 	// Tags must be letters, digits, underscores or dots.
 	// Unlike in Go identifiers, all digits are fine (e.g., "386").
 	for _, c := range name {
@@ -116,13 +126,22 @@
 		}
 	}
 
-	if name == "linux" && tags["android"] {
+	if tags["*"] && name != "" && name != "ignore" {
+		// Special case for gathering all possible imports:
+		// if we put * in the tags map then all tags
+		// except "ignore" are considered both present and not
+		// (so we return true no matter how 'want' is set).
 		return true
 	}
-	return tags[name]
+
+	have := tags[name]
+	if name == "linux" {
+		have = have || tags["android"]
+	}
+	return have == want
 }
 
-// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH
+// MatchFile returns false if the name contains a $GOOS or $GOARCH
 // suffix which does not match the current system.
 // The recognized name formats are:
 //
@@ -134,7 +153,14 @@
 //     name_$(GOOS)_$(GOARCH)_test.*
 //
 // An exception: if GOOS=android, then files with GOOS=linux are also matched.
+//
+// If tags["*"] is true, then MatchFile will consider all possible
+// GOOS and GOARCH to be available and will consequently
+// always return true.
 func MatchFile(name string, tags map[string]bool) bool {
+	if tags["*"] {
+		return true
+	}
 	if dot := strings.Index(name, "."); dot != -1 {
 		name = name[:dot]
 	}

diff --git a/vendor/cmd/go/internal/imports/scan_test.go b/vendor/cmd/go/internal/imports/scan_test.go
index cb0a46a..dbc7944 100644
--- a/vendor/cmd/go/internal/imports/scan_test.go
+++ b/vendor/cmd/go/internal/imports/scan_test.go

@@ -6,6 +6,7 @@
 
 import (
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"testing"
 )
@@ -47,3 +48,15 @@
 		t.Errorf("json missing test import net/http (%q)", testImports)
 	}
 }
+
+func TestScanStar(t *testing.T) {
+	imports, _, err := ScanDir("testdata/import1", map[string]bool{"*": true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := []string{"import1", "import2", "import3", "import4"}
+	if !reflect.DeepEqual(imports, want) {
+		t.Errorf("ScanDir testdata/import1:\nhave %v\nwant %v", imports, want)
+	}
+}

diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x.go b/vendor/cmd/go/internal/imports/testdata/import1/x.go
new file mode 100644
index 0000000..98f9191
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x.go

@@ -0,0 +1,3 @@
+package x
+
+import "import1"

diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x1.go b/vendor/cmd/go/internal/imports/testdata/import1/x1.go
new file mode 100644
index 0000000..6a9594a
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x1.go

@@ -0,0 +1,9 @@
+// +build blahblh
+// +build linux
+// +build !linux
+// +build windows
+// +build darwin
+
+package x
+
+import "import4"

diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go b/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go
new file mode 100644
index 0000000..a0c3fdd
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go

@@ -0,0 +1,3 @@
+package xxxx
+
+import "import3"

diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go b/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go
new file mode 100644
index 0000000..63c5082
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go

@@ -0,0 +1,3 @@
+package x
+
+import "import2"

diff --git a/vendor/cmd/go/internal/vgo/load.go b/vendor/cmd/go/internal/vgo/load.go
index 0f11570..1b14e4b 100644
--- a/vendor/cmd/go/internal/vgo/load.go
+++ b/vendor/cmd/go/internal/vgo/load.go

@@ -41,6 +41,7 @@
 	tags      map[string]bool
 	importmap map[string]string
 	pkgdir    map[string]string
+	pkgmod    map[string]module.Version
 	isGetU    bool
 )
 
@@ -146,12 +147,15 @@
 
 	importmap = ld.importmap
 	pkgdir = ld.pkgdir
+	pkgmod = ld.pkgmod
 }
 
 type loader struct {
 	imported  map[string]importLevel
 	importmap map[string]string
 	pkgdir    map[string]string
+	pkgmod    map[string]module.Version
+	tags      map[string]bool
 	missing   []missing
 	imports   []string
 	stack     []string
@@ -167,6 +171,8 @@
 		imported:  make(map[string]importLevel),
 		importmap: make(map[string]string),
 		pkgdir:    make(map[string]string),
+		pkgmod:    make(map[string]module.Version),
+		tags:      imports.Tags(),
 	}
 	ld.imported["C"] = 100
 	return ld
@@ -214,7 +220,7 @@
 
 	ld.pkgdir[realPath] = dir
 
-	imports, testImports, err := imports.ScanDir(dir, imports.Tags())
+	imports, testImports, err := imports.ScanDir(dir, ld.tags)
 	if err != nil {
 		base.Errorf("vgo: %s [%s]: %v", ld.stackText(), dir, err)
 		return
@@ -239,6 +245,7 @@
 		if len(path) > len(Target.Path) {
 			dir = filepath.Join(dir, path[len(Target.Path)+1:])
 		}
+		ld.pkgmod[path] = Target
 		return dir
 	}
 
@@ -276,6 +283,7 @@
 		mod1 = mod
 	}
 	if dir1 != "" {
+		ld.pkgmod[path] = mod1
 		return dir1
 	}
 	ld.missing = append(ld.missing, missing{path, ld.stackText()})

diff --git a/vendor/cmd/go/internal/vgo/search.go b/vendor/cmd/go/internal/vgo/search.go
index 13a426c..c3f7ab1 100644
--- a/vendor/cmd/go/internal/vgo/search.go
+++ b/vendor/cmd/go/internal/vgo/search.go

@@ -22,11 +22,13 @@
 func expandImportPaths(args []string) []string {
 	var out []string
 	for _, a := range args {
-		if search.IsMetaPackage(a) {
+		// TODO(rsc): Move a == "ALL" test into search.IsMetaPackage
+		// once we officially lock in all the module work (tentatively, Go 1.12).
+		if search.IsMetaPackage(a) || a == "ALL" {
 			switch a {
 			default:
-				fmt.Fprintf(os.Stderr, "warning: %q matches no packages when using modules", a)
-			case "all":
+				fmt.Fprintf(os.Stderr, "vgo: warning: %q matches no packages when using modules\n", a)
+			case "all", "ALL":
 				out = append(out, AllPackages(a)...)
 			}
 			continue
@@ -65,6 +67,9 @@
 	if pattern == "all" {
 		return MatchAll()
 	}
+	if pattern == "ALL" {
+		return MatchALL()
+	}
 
 	return matchPackages(pattern, buildList)
 }
@@ -72,7 +77,7 @@
 func matchPackages(pattern string, buildList []module.Version) []string {
 	match := func(string) bool { return true }
 	treeCanMatch := func(string) bool { return true }
-	if !search.IsMetaPackage(pattern) {
+	if !search.IsMetaPackage(pattern) && pattern != "ALL" {
 		match = search.MatchPattern(pattern)
 		treeCanMatch = search.TreeCanMatchPattern(pattern)
 	}
@@ -162,8 +167,23 @@
 // It does not include packages in other modules that are not needed
 // by builds of this module.
 func MatchAll() []string {
+	return matchAll(imports.Tags())
+}
+
+// MatchALL returns a list of the packages matching the pattern "ALL".
+// The pattern "ALL" is like "all" but looks at all source files,
+// even ones that would be ignored by current build tag settings.
+// That's useful for identifying which packages to include in a vendor directory.
+func MatchALL() []string {
+	return matchAll(map[string]bool{"*": true})
+}
+
+// matchAll is the common implementation of MatchAll and MatchALL,
+// which differ only in the set of tags to apply to select files.
+func matchAll(tags map[string]bool) []string {
 	local := matchPackages("all", buildList[:1])
 	ld := newLoader()
+	ld.tags = tags
 	ld.importList(local, levelTestRecursive)
 	var all []string
 	for _, pkg := range ld.importmap {

diff --git a/vendor/cmd/go/internal/vgo/vendor.go b/vendor/cmd/go/internal/vgo/vendor.go
index e3330f6..acba4af 100644
--- a/vendor/cmd/go/internal/vgo/vendor.go
+++ b/vendor/cmd/go/internal/vgo/vendor.go

@@ -14,19 +14,29 @@
 	"strings"
 
 	"cmd/go/internal/base"
+	"cmd/go/internal/module"
 )
 
 var CmdVendor = &base.Command{
-	UsageLine: "vendor",
-	Run:       runVendor,
+	UsageLine: "vendor [-v]",
 	Short:     "vendor dependencies of current module",
 	Long: `
 Vendor resets the module's vendor directory to include all
-packages needed to builds and test all packages in the module
+packages needed to build and test all packages in the module
 and their dependencies.
+
+The -v flag causes vendor to print to standard error the
+module paths of the modules processed and the import paths
+of the packages copied.
 	`,
 }
 
+var vendorV = CmdVendor.Flag.Bool("v", false, "")
+
+func init() {
+	CmdVendor.Run = runVendor // break init cycle
+}
+
 func runVendor(cmd *base.Command, args []string) {
 	if Init(); !Enabled() {
 		base.Fatalf("vgo vendor: cannot use -m outside module")
@@ -35,29 +45,51 @@
 		base.Fatalf("vgo vendor: vendor takes no arguments")
 	}
 	InitMod()
-	// TODO(rsc): This should scan directories with more permissive build tags.
-	pkgs := ImportPaths([]string{"all"})
+	pkgs := ImportPaths([]string{"ALL"})
 
 	vdir := filepath.Join(ModRoot, "vendor")
 	if err := os.RemoveAll(vdir); err != nil {
 		base.Fatalf("vgo vendor: %v", err)
 	}
 
+	modpkgs := make(map[module.Version][]string)
 	for _, pkg := range pkgs {
-		vendorPkg(vdir, pkg)
+		m := pkgmod[pkg]
+		if m == Target {
+			continue
+		}
+		modpkgs[m] = append(modpkgs[m], pkg)
 	}
 
 	var buf bytes.Buffer
-	printListM(&buf)
+	for _, m := range buildList[1:] {
+		if pkgs := modpkgs[m]; len(pkgs) > 0 {
+			repl := ""
+			if r := replaced(m); r != nil {
+				repl = " => " + r.New.Path
+				if r.New.Version != "" {
+					repl += " " + r.New.Version
+				}
+			}
+			fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
+			if *vendorV {
+				fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
+			}
+			for _, pkg := range pkgs {
+				fmt.Fprintf(&buf, "%s\n", pkg)
+				if *vendorV {
+					fmt.Fprintf(os.Stderr, "%s\n", pkg)
+				}
+				vendorPkg(vdir, pkg)
+			}
+		}
+	}
 	if err := ioutil.WriteFile(filepath.Join(vdir, "vgo.list"), buf.Bytes(), 0666); err != nil {
 		base.Fatalf("vgo vendor: %v", err)
 	}
 }
 
 func vendorPkg(vdir, pkg string) {
-	if hasPathPrefix(pkg, Target.Path) {
-		return
-	}
 	realPath := importmap[pkg]
 	if realPath != pkg && importmap[realPath] != "" {
 		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)

diff --git a/vendor/cmd/go/testdata/vendormod/go.mod b/vendor/cmd/go/testdata/vendormod/go.mod
new file mode 100644
index 0000000..6f71634
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/go.mod

@@ -0,0 +1,16 @@
+module m
+
+replace x v1.0.0 => ./x
+
+replace y v1.0.0 => ./y
+
+replace z v1.0.0 => ./z
+
+replace w v1.0.0 => ./w
+
+require (
+	w v1.0.0
+	x v1.0.0
+	y v1.0.0
+	z v1.0.0
+)

diff --git a/vendor/cmd/go/testdata/vendormod/v1.go b/vendor/cmd/go/testdata/vendormod/v1.go
new file mode 100644
index 0000000..6ca04a5
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v1.go

@@ -0,0 +1,3 @@
+package m
+
+import _ "x"

diff --git a/vendor/cmd/go/testdata/vendormod/v2.go b/vendor/cmd/go/testdata/vendormod/v2.go
new file mode 100644
index 0000000..8b089e4
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v2.go

@@ -0,0 +1,5 @@
+// +build abc
+
+package mMmMmMm
+
+import _ "y"

diff --git a/vendor/cmd/go/testdata/vendormod/v3.go b/vendor/cmd/go/testdata/vendormod/v3.go
new file mode 100644
index 0000000..318b5f0
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v3.go

@@ -0,0 +1,5 @@
+// +build !abc
+
+package m
+
+import _ "z"

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/vgo.list b/vendor/cmd/go/testdata/vendormod/vendor/vgo.list
new file mode 100644
index 0000000..e8bd2c5
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/vgo.list

@@ -0,0 +1,6 @@
+# x v1.0.0 => ./x
+x
+# y v1.0.0 => ./y
+y
+# z v1.0.0 => ./z
+z

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod
new file mode 100644
index 0000000..c191435
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod

@@ -0,0 +1 @@
+module x

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/x/x.go b/vendor/cmd/go/testdata/vendormod/vendor/x/x.go
new file mode 100644
index 0000000..823aafd
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/x/x.go

@@ -0,0 +1 @@
+package x

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod
new file mode 100644
index 0000000..ac82a48
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod

@@ -0,0 +1 @@
+module y

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/y/y.go b/vendor/cmd/go/testdata/vendormod/vendor/y/y.go
new file mode 100644
index 0000000..789ca71
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/y/y.go

@@ -0,0 +1 @@
+package y

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod
new file mode 100644
index 0000000..efc58fe
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod

@@ -0,0 +1 @@
+module z

diff --git a/vendor/cmd/go/testdata/vendormod/vendor/z/z.go b/vendor/cmd/go/testdata/vendormod/vendor/z/z.go
new file mode 100644
index 0000000..46458cb
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/z/z.go

@@ -0,0 +1 @@
+package z

diff --git a/vendor/cmd/go/testdata/vendormod/w/go.mod b/vendor/cmd/go/testdata/vendormod/w/go.mod
new file mode 100644
index 0000000..ce2a6c1
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/w/go.mod

@@ -0,0 +1 @@
+module w

diff --git a/vendor/cmd/go/testdata/vendormod/w/w.go b/vendor/cmd/go/testdata/vendormod/w/w.go
new file mode 100644
index 0000000..a796c0b
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/w/w.go

@@ -0,0 +1 @@
+package w

diff --git a/vendor/cmd/go/testdata/vendormod/x/go.mod b/vendor/cmd/go/testdata/vendormod/x/go.mod
new file mode 100644
index 0000000..c191435
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/x/go.mod

@@ -0,0 +1 @@
+module x

diff --git a/vendor/cmd/go/testdata/vendormod/x/x.go b/vendor/cmd/go/testdata/vendormod/x/x.go
new file mode 100644
index 0000000..823aafd
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/x/x.go

@@ -0,0 +1 @@
+package x

diff --git a/vendor/cmd/go/testdata/vendormod/y/go.mod b/vendor/cmd/go/testdata/vendormod/y/go.mod
new file mode 100644
index 0000000..ac82a48
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/y/go.mod

@@ -0,0 +1 @@
+module y

diff --git a/vendor/cmd/go/testdata/vendormod/y/y.go b/vendor/cmd/go/testdata/vendormod/y/y.go
new file mode 100644
index 0000000..789ca71
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/y/y.go

@@ -0,0 +1 @@
+package y

diff --git a/vendor/cmd/go/testdata/vendormod/z/go.mod b/vendor/cmd/go/testdata/vendormod/z/go.mod
new file mode 100644
index 0000000..efc58fe
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/z/go.mod

@@ -0,0 +1 @@
+module z

diff --git a/vendor/cmd/go/testdata/vendormod/z/z.go b/vendor/cmd/go/testdata/vendormod/z/z.go
new file mode 100644
index 0000000..46458cb
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/z/z.go

@@ -0,0 +1 @@
+package z

diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go
index 6d7032a..f559970 100644
--- a/vendor/cmd/go/vgo_test.go
+++ b/vendor/cmd/go/vgo_test.go

@@ -221,6 +221,29 @@
 	tg.grepStderr("tcp.*nonexistent.rsc.io", "expected error for nonexistent.rsc.io")
 }
 
+func TestVgoVendor(t *testing.T) {
+	tg := testgo(t)
+	defer tg.cleanup()
+
+	wd, _ := os.Getwd()
+	tg.cd(filepath.Join(wd, "testdata/vendormod"))
+	tg.run("-vgo", "list", "-m")
+	tg.grepStdout(`^x`, "expected to see module x")
+	tg.grepStdout(`=> ./x`, "expected to see replacement for module x")
+	tg.grepStdout(`^w`, "expected to see module w")
+
+	tg.run("-vgo", "vendor", "-v")
+	tg.grepStderr(`^# x v1.0.0 => ./x`, "expected to see module x with replacement")
+	tg.grepStderr(`^x`, "expected to see package x")
+	tg.grepStderr(`^# y v1.0.0 => ./y`, "expected to see module y with replacement")
+	tg.grepStderr(`^y`, "expected to see package y")
+	tg.grepStderr(`^# z v1.0.0 => ./z`, "expected to see module z with replacement")
+	tg.grepStderr(`^z`, "expected to see package z")
+	tg.grepStderrNot(`w`, "expected NOT to see unused module w")
+
+	tg.must(os.RemoveAll(filepath.Join(wd, "testdata/vendormod/vendor")))
+}
+
 func TestFillGoMod(t *testing.T) {
 	testenv.MustHaveExternalNetwork(t)
 	tg := testgo(t)