cmd/go/internal/vgo: process packages for all build tags in vendor
Until now, "vgo vendor" has only considered the packages needed
for a build with the current GOOS/GOARCH and build tags.
If a module has both linux-specific and windows-specific
requirements, then "vgo vendor" on windows will omit the
linux code, and vice versa, making it impossible to prepare
a full vendor set.
This CL introduces a new (for now undocumented) package
pattern "ALL" that is like "all" but follows imports in files
requiring any plausible build tag, producing a set of files
that does not vary from system to system.
The only +build lines not satisfied when computing "ALL"
packages are those listing only the tag "ignore" or those
listing only syntactically invalid build tags. For example,
a file containing any of these lines not be processed:
// +build ignore
// +build !
// +build @#$%^&*()
(The first two are commonly used in practice to exclude
a file from processing by the go command.)
This CL then changes the "vgo vendor" command to
process "ALL" packages, not "all" packages.
It also changes the vendor/vgo.list output file to list only
modules that had files copied out of them and to indicate
which packages came from which modules.
Fixes golang/go#25376.
Fixes golang/go#25624.
Change-Id: I6756286fa6bb697819279ef67b4bea68bc0b7f53
Reviewed-on: https://go-review.googlesource.com/117258
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/vendor/cmd/go/internal/imports/build.go b/vendor/cmd/go/internal/imports/build.go
index bb40f2b..5597870 100644
--- a/vendor/cmd/go/internal/imports/build.go
+++ b/vendor/cmd/go/internal/imports/build.go
@@ -27,6 +27,12 @@
//
// marks the file as applicable only on Windows and Linux.
//
+// If tags["*"] is true, then ShouldBuild will consider every
+// build tag except "ignore" to be both true and false for
+// the purpose of satisfying build tags, in order to estimate
+// (conservatively) whether a file could ever possibly be used
+// in any build.
+//
func ShouldBuild(content []byte, tags map[string]bool) bool {
// Pass 1. Identify leading run of // comments and blank lines,
// which must be followed by a blank line.
@@ -87,8 +93,8 @@
// matchTags reports whether the name is one of:
//
-// tag (if haveTags[tag] is true)
-// !tag (if haveTags[tag] is false)
+// tag (if tags[tag] is true)
+// !tag (if tags[tag] is false)
// a comma-separated list of any of these
//
func matchTags(name string, tags map[string]bool) bool {
@@ -105,9 +111,13 @@
return false
}
if strings.HasPrefix(name, "!") { // negation
- return len(name) > 1 && !matchTags(name[1:], tags)
+ return len(name) > 1 && matchTag(name[1:], tags, false)
}
+ return matchTag(name, tags, true)
+}
+// matchTag reports whether the tag name is valid and satisfied by tags[name]==want.
+func matchTag(name string, tags map[string]bool, want bool) bool {
// Tags must be letters, digits, underscores or dots.
// Unlike in Go identifiers, all digits are fine (e.g., "386").
for _, c := range name {
@@ -116,13 +126,22 @@
}
}
- if name == "linux" && tags["android"] {
+ if tags["*"] && name != "" && name != "ignore" {
+ // Special case for gathering all possible imports:
+ // if we put * in the tags map then all tags
+ // except "ignore" are considered both present and not
+ // (so we return true no matter how 'want' is set).
return true
}
- return tags[name]
+
+ have := tags[name]
+ if name == "linux" {
+ have = have || tags["android"]
+ }
+ return have == want
}
-// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH
+// MatchFile returns false if the name contains a $GOOS or $GOARCH
// suffix which does not match the current system.
// The recognized name formats are:
//
@@ -134,7 +153,14 @@
// name_$(GOOS)_$(GOARCH)_test.*
//
// An exception: if GOOS=android, then files with GOOS=linux are also matched.
+//
+// If tags["*"] is true, then MatchFile will consider all possible
+// GOOS and GOARCH to be available and will consequently
+// always return true.
func MatchFile(name string, tags map[string]bool) bool {
+ if tags["*"] {
+ return true
+ }
if dot := strings.Index(name, "."); dot != -1 {
name = name[:dot]
}
diff --git a/vendor/cmd/go/internal/imports/scan_test.go b/vendor/cmd/go/internal/imports/scan_test.go
index cb0a46a..dbc7944 100644
--- a/vendor/cmd/go/internal/imports/scan_test.go
+++ b/vendor/cmd/go/internal/imports/scan_test.go
@@ -6,6 +6,7 @@
import (
"path/filepath"
+ "reflect"
"runtime"
"testing"
)
@@ -47,3 +48,15 @@
t.Errorf("json missing test import net/http (%q)", testImports)
}
}
+
+func TestScanStar(t *testing.T) {
+ imports, _, err := ScanDir("testdata/import1", map[string]bool{"*": true})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ want := []string{"import1", "import2", "import3", "import4"}
+ if !reflect.DeepEqual(imports, want) {
+ t.Errorf("ScanDir testdata/import1:\nhave %v\nwant %v", imports, want)
+ }
+}
diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x.go b/vendor/cmd/go/internal/imports/testdata/import1/x.go
new file mode 100644
index 0000000..98f9191
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x.go
@@ -0,0 +1,3 @@
+package x
+
+import "import1"
diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x1.go b/vendor/cmd/go/internal/imports/testdata/import1/x1.go
new file mode 100644
index 0000000..6a9594a
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x1.go
@@ -0,0 +1,9 @@
+// +build blahblh
+// +build linux
+// +build !linux
+// +build windows
+// +build darwin
+
+package x
+
+import "import4"
diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go b/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go
new file mode 100644
index 0000000..a0c3fdd
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x_darwin.go
@@ -0,0 +1,3 @@
+package xxxx
+
+import "import3"
diff --git a/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go b/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go
new file mode 100644
index 0000000..63c5082
--- /dev/null
+++ b/vendor/cmd/go/internal/imports/testdata/import1/x_windows.go
@@ -0,0 +1,3 @@
+package x
+
+import "import2"
diff --git a/vendor/cmd/go/internal/vgo/load.go b/vendor/cmd/go/internal/vgo/load.go
index 0f11570..1b14e4b 100644
--- a/vendor/cmd/go/internal/vgo/load.go
+++ b/vendor/cmd/go/internal/vgo/load.go
@@ -41,6 +41,7 @@
tags map[string]bool
importmap map[string]string
pkgdir map[string]string
+ pkgmod map[string]module.Version
isGetU bool
)
@@ -146,12 +147,15 @@
importmap = ld.importmap
pkgdir = ld.pkgdir
+ pkgmod = ld.pkgmod
}
type loader struct {
imported map[string]importLevel
importmap map[string]string
pkgdir map[string]string
+ pkgmod map[string]module.Version
+ tags map[string]bool
missing []missing
imports []string
stack []string
@@ -167,6 +171,8 @@
imported: make(map[string]importLevel),
importmap: make(map[string]string),
pkgdir: make(map[string]string),
+ pkgmod: make(map[string]module.Version),
+ tags: imports.Tags(),
}
ld.imported["C"] = 100
return ld
@@ -214,7 +220,7 @@
ld.pkgdir[realPath] = dir
- imports, testImports, err := imports.ScanDir(dir, imports.Tags())
+ imports, testImports, err := imports.ScanDir(dir, ld.tags)
if err != nil {
base.Errorf("vgo: %s [%s]: %v", ld.stackText(), dir, err)
return
@@ -239,6 +245,7 @@
if len(path) > len(Target.Path) {
dir = filepath.Join(dir, path[len(Target.Path)+1:])
}
+ ld.pkgmod[path] = Target
return dir
}
@@ -276,6 +283,7 @@
mod1 = mod
}
if dir1 != "" {
+ ld.pkgmod[path] = mod1
return dir1
}
ld.missing = append(ld.missing, missing{path, ld.stackText()})
diff --git a/vendor/cmd/go/internal/vgo/search.go b/vendor/cmd/go/internal/vgo/search.go
index 13a426c..c3f7ab1 100644
--- a/vendor/cmd/go/internal/vgo/search.go
+++ b/vendor/cmd/go/internal/vgo/search.go
@@ -22,11 +22,13 @@
func expandImportPaths(args []string) []string {
var out []string
for _, a := range args {
- if search.IsMetaPackage(a) {
+ // TODO(rsc): Move a == "ALL" test into search.IsMetaPackage
+ // once we officially lock in all the module work (tentatively, Go 1.12).
+ if search.IsMetaPackage(a) || a == "ALL" {
switch a {
default:
- fmt.Fprintf(os.Stderr, "warning: %q matches no packages when using modules", a)
- case "all":
+ fmt.Fprintf(os.Stderr, "vgo: warning: %q matches no packages when using modules\n", a)
+ case "all", "ALL":
out = append(out, AllPackages(a)...)
}
continue
@@ -65,6 +67,9 @@
if pattern == "all" {
return MatchAll()
}
+ if pattern == "ALL" {
+ return MatchALL()
+ }
return matchPackages(pattern, buildList)
}
@@ -72,7 +77,7 @@
func matchPackages(pattern string, buildList []module.Version) []string {
match := func(string) bool { return true }
treeCanMatch := func(string) bool { return true }
- if !search.IsMetaPackage(pattern) {
+ if !search.IsMetaPackage(pattern) && pattern != "ALL" {
match = search.MatchPattern(pattern)
treeCanMatch = search.TreeCanMatchPattern(pattern)
}
@@ -162,8 +167,23 @@
// It does not include packages in other modules that are not needed
// by builds of this module.
func MatchAll() []string {
+ return matchAll(imports.Tags())
+}
+
+// MatchALL returns a list of the packages matching the pattern "ALL".
+// The pattern "ALL" is like "all" but looks at all source files,
+// even ones that would be ignored by current build tag settings.
+// That's useful for identifying which packages to include in a vendor directory.
+func MatchALL() []string {
+ return matchAll(map[string]bool{"*": true})
+}
+
+// matchAll is the common implementation of MatchAll and MatchALL,
+// which differ only in the set of tags to apply to select files.
+func matchAll(tags map[string]bool) []string {
local := matchPackages("all", buildList[:1])
ld := newLoader()
+ ld.tags = tags
ld.importList(local, levelTestRecursive)
var all []string
for _, pkg := range ld.importmap {
diff --git a/vendor/cmd/go/internal/vgo/vendor.go b/vendor/cmd/go/internal/vgo/vendor.go
index e3330f6..acba4af 100644
--- a/vendor/cmd/go/internal/vgo/vendor.go
+++ b/vendor/cmd/go/internal/vgo/vendor.go
@@ -14,19 +14,29 @@
"strings"
"cmd/go/internal/base"
+ "cmd/go/internal/module"
)
var CmdVendor = &base.Command{
- UsageLine: "vendor",
- Run: runVendor,
+ UsageLine: "vendor [-v]",
Short: "vendor dependencies of current module",
Long: `
Vendor resets the module's vendor directory to include all
-packages needed to builds and test all packages in the module
+packages needed to build and test all packages in the module
and their dependencies.
+
+The -v flag causes vendor to print to standard error the
+module paths of the modules processed and the import paths
+of the packages copied.
`,
}
+var vendorV = CmdVendor.Flag.Bool("v", false, "")
+
+func init() {
+ CmdVendor.Run = runVendor // break init cycle
+}
+
func runVendor(cmd *base.Command, args []string) {
if Init(); !Enabled() {
base.Fatalf("vgo vendor: cannot use -m outside module")
@@ -35,29 +45,51 @@
base.Fatalf("vgo vendor: vendor takes no arguments")
}
InitMod()
- // TODO(rsc): This should scan directories with more permissive build tags.
- pkgs := ImportPaths([]string{"all"})
+ pkgs := ImportPaths([]string{"ALL"})
vdir := filepath.Join(ModRoot, "vendor")
if err := os.RemoveAll(vdir); err != nil {
base.Fatalf("vgo vendor: %v", err)
}
+ modpkgs := make(map[module.Version][]string)
for _, pkg := range pkgs {
- vendorPkg(vdir, pkg)
+ m := pkgmod[pkg]
+ if m == Target {
+ continue
+ }
+ modpkgs[m] = append(modpkgs[m], pkg)
}
var buf bytes.Buffer
- printListM(&buf)
+ for _, m := range buildList[1:] {
+ if pkgs := modpkgs[m]; len(pkgs) > 0 {
+ repl := ""
+ if r := replaced(m); r != nil {
+ repl = " => " + r.New.Path
+ if r.New.Version != "" {
+ repl += " " + r.New.Version
+ }
+ }
+ fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
+ if *vendorV {
+ fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
+ }
+ for _, pkg := range pkgs {
+ fmt.Fprintf(&buf, "%s\n", pkg)
+ if *vendorV {
+ fmt.Fprintf(os.Stderr, "%s\n", pkg)
+ }
+ vendorPkg(vdir, pkg)
+ }
+ }
+ }
if err := ioutil.WriteFile(filepath.Join(vdir, "vgo.list"), buf.Bytes(), 0666); err != nil {
base.Fatalf("vgo vendor: %v", err)
}
}
func vendorPkg(vdir, pkg string) {
- if hasPathPrefix(pkg, Target.Path) {
- return
- }
realPath := importmap[pkg]
if realPath != pkg && importmap[realPath] != "" {
fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
diff --git a/vendor/cmd/go/testdata/vendormod/go.mod b/vendor/cmd/go/testdata/vendormod/go.mod
new file mode 100644
index 0000000..6f71634
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/go.mod
@@ -0,0 +1,16 @@
+module m
+
+replace x v1.0.0 => ./x
+
+replace y v1.0.0 => ./y
+
+replace z v1.0.0 => ./z
+
+replace w v1.0.0 => ./w
+
+require (
+ w v1.0.0
+ x v1.0.0
+ y v1.0.0
+ z v1.0.0
+)
diff --git a/vendor/cmd/go/testdata/vendormod/v1.go b/vendor/cmd/go/testdata/vendormod/v1.go
new file mode 100644
index 0000000..6ca04a5
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v1.go
@@ -0,0 +1,3 @@
+package m
+
+import _ "x"
diff --git a/vendor/cmd/go/testdata/vendormod/v2.go b/vendor/cmd/go/testdata/vendormod/v2.go
new file mode 100644
index 0000000..8b089e4
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v2.go
@@ -0,0 +1,5 @@
+// +build abc
+
+package mMmMmMm
+
+import _ "y"
diff --git a/vendor/cmd/go/testdata/vendormod/v3.go b/vendor/cmd/go/testdata/vendormod/v3.go
new file mode 100644
index 0000000..318b5f0
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/v3.go
@@ -0,0 +1,5 @@
+// +build !abc
+
+package m
+
+import _ "z"
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/vgo.list b/vendor/cmd/go/testdata/vendormod/vendor/vgo.list
new file mode 100644
index 0000000..e8bd2c5
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/vgo.list
@@ -0,0 +1,6 @@
+# x v1.0.0 => ./x
+x
+# y v1.0.0 => ./y
+y
+# z v1.0.0 => ./z
+z
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod
new file mode 100644
index 0000000..c191435
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/x/go.mod
@@ -0,0 +1 @@
+module x
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/x/x.go b/vendor/cmd/go/testdata/vendormod/vendor/x/x.go
new file mode 100644
index 0000000..823aafd
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/x/x.go
@@ -0,0 +1 @@
+package x
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod
new file mode 100644
index 0000000..ac82a48
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/y/go.mod
@@ -0,0 +1 @@
+module y
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/y/y.go b/vendor/cmd/go/testdata/vendormod/vendor/y/y.go
new file mode 100644
index 0000000..789ca71
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/y/y.go
@@ -0,0 +1 @@
+package y
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod b/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod
new file mode 100644
index 0000000..efc58fe
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/z/go.mod
@@ -0,0 +1 @@
+module z
diff --git a/vendor/cmd/go/testdata/vendormod/vendor/z/z.go b/vendor/cmd/go/testdata/vendormod/vendor/z/z.go
new file mode 100644
index 0000000..46458cb
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/vendor/z/z.go
@@ -0,0 +1 @@
+package z
diff --git a/vendor/cmd/go/testdata/vendormod/w/go.mod b/vendor/cmd/go/testdata/vendormod/w/go.mod
new file mode 100644
index 0000000..ce2a6c1
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/w/go.mod
@@ -0,0 +1 @@
+module w
diff --git a/vendor/cmd/go/testdata/vendormod/w/w.go b/vendor/cmd/go/testdata/vendormod/w/w.go
new file mode 100644
index 0000000..a796c0b
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/w/w.go
@@ -0,0 +1 @@
+package w
diff --git a/vendor/cmd/go/testdata/vendormod/x/go.mod b/vendor/cmd/go/testdata/vendormod/x/go.mod
new file mode 100644
index 0000000..c191435
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/x/go.mod
@@ -0,0 +1 @@
+module x
diff --git a/vendor/cmd/go/testdata/vendormod/x/x.go b/vendor/cmd/go/testdata/vendormod/x/x.go
new file mode 100644
index 0000000..823aafd
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/x/x.go
@@ -0,0 +1 @@
+package x
diff --git a/vendor/cmd/go/testdata/vendormod/y/go.mod b/vendor/cmd/go/testdata/vendormod/y/go.mod
new file mode 100644
index 0000000..ac82a48
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/y/go.mod
@@ -0,0 +1 @@
+module y
diff --git a/vendor/cmd/go/testdata/vendormod/y/y.go b/vendor/cmd/go/testdata/vendormod/y/y.go
new file mode 100644
index 0000000..789ca71
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/y/y.go
@@ -0,0 +1 @@
+package y
diff --git a/vendor/cmd/go/testdata/vendormod/z/go.mod b/vendor/cmd/go/testdata/vendormod/z/go.mod
new file mode 100644
index 0000000..efc58fe
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/z/go.mod
@@ -0,0 +1 @@
+module z
diff --git a/vendor/cmd/go/testdata/vendormod/z/z.go b/vendor/cmd/go/testdata/vendormod/z/z.go
new file mode 100644
index 0000000..46458cb
--- /dev/null
+++ b/vendor/cmd/go/testdata/vendormod/z/z.go
@@ -0,0 +1 @@
+package z
diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go
index 6d7032a..f559970 100644
--- a/vendor/cmd/go/vgo_test.go
+++ b/vendor/cmd/go/vgo_test.go
@@ -221,6 +221,29 @@
tg.grepStderr("tcp.*nonexistent.rsc.io", "expected error for nonexistent.rsc.io")
}
+func TestVgoVendor(t *testing.T) {
+ tg := testgo(t)
+ defer tg.cleanup()
+
+ wd, _ := os.Getwd()
+ tg.cd(filepath.Join(wd, "testdata/vendormod"))
+ tg.run("-vgo", "list", "-m")
+ tg.grepStdout(`^x`, "expected to see module x")
+ tg.grepStdout(`=> ./x`, "expected to see replacement for module x")
+ tg.grepStdout(`^w`, "expected to see module w")
+
+ tg.run("-vgo", "vendor", "-v")
+ tg.grepStderr(`^# x v1.0.0 => ./x`, "expected to see module x with replacement")
+ tg.grepStderr(`^x`, "expected to see package x")
+ tg.grepStderr(`^# y v1.0.0 => ./y`, "expected to see module y with replacement")
+ tg.grepStderr(`^y`, "expected to see package y")
+ tg.grepStderr(`^# z v1.0.0 => ./z`, "expected to see module z with replacement")
+ tg.grepStderr(`^z`, "expected to see package z")
+ tg.grepStderrNot(`w`, "expected NOT to see unused module w")
+
+ tg.must(os.RemoveAll(filepath.Join(wd, "testdata/vendormod/vendor")))
+}
+
func TestFillGoMod(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
tg := testgo(t)
