cmd/go/internal/vgo: track directly-used vs indirectly-used modules A cleanup pass in mvs.BuildList discards modules that are not reachable in the requirement graph as satisfied for this build. For example, suppose: A -> B1, C1 B1 -> D1 B2 -> nothing C1 -> nothing D1 -> nothing D2 -> nothing The effective build list is A, B1, C1, D1 (no cleanup possible). Suppose that we update from B1 to B2. The effective build list becomes A, B2, C1, D1, and since there is no path through those module versions from A to D, the cleanup pass drops D. This cleanup, which is not in https://research.swtch.com/vgo-mvs, aims to avoid user confusion by not listing irrelevant modules in the output of commands like "vgo list -m all". Unfortunately, the cleanup is not sound in general, because there is no guarantee all of A's needs are listed as direct requirements. For example, maybe A imports D. In that case, dropping D and then building A will re-add the latest version of D (D2 instead of D1). The most common time this happens is after an upgrade. The fix is to make sure that go.mod does list all of the modules required directly by A, and to make sure that the go.mod minimizer (Algorithm R in the blog post) does not remove direct requirements in the name of simplifying go.mod. The way this is done is to annotate the requirements NOT used directly by A with a known comment, "// indirect". For example suppose A imports rsc.io/quote. Then the go.mod looks like it always has: module m require rsc.io/quote v1.5.2 But now suppose we upgrade our packages to their latest versions. Then go.mod becomes: module m require ( golang.org/x/text v0.3.0 // indirect rsc.io/quote v1.5.2 rsc.io/sampler v1.99.99 // indirect ) The "// indirect" comments indicate that this requirement is used only to upgrade something needed outside module m, not to satisfy any packages in module m itself. Vgo adds and removes these comments automatically. If we add a direct import of golang.org/x/text to some package in m, then the first time we build that package vgo strips the "// indirect" on the golang.org/x/text requirement line. If we then remove that package, the requirement remains listed as direct (the conservative choice) until the next "vgo mod -sync", which considers all packages in m and can mark the requirement indirect again. Algorithm R is modified to be given a set of import paths that must be preserved in the final output (all the ones not marked // indirect). Maintenance of this extra information makes the cleanup pass safe. Seeing all directly-imported modules in go.mod and distinguishing between directly- and indirectly-imported modules in go.mod are two of the most commonly-requested features, so it's extra nice that the fix for the cleanup-induced bug makes go.mod closer to what users expect. Fixes golang/go#24042. Fixes golang/go#25371. Fixes golang/go#25969. Change-Id: I4ed0729b867723fe90e836c2325f740b55b2b27b Reviewed-on: https://go-review.googlesource.com/121304 Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/vendor/cmd/go/internal/list/list.go b/vendor/cmd/go/internal/list/list.go index ad078ee..c5f0550 100644 --- a/vendor/cmd/go/internal/list/list.go +++ b/vendor/cmd/go/internal/list/list.go
@@ -199,6 +199,7 @@ Time *time.Time // time version was created Update *Module // available update, if any (with -u) Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? Dir string // directory holding files for this module, if any Error *ModuleError // error loading module }
diff --git a/vendor/cmd/go/internal/modcmd/mod.go b/vendor/cmd/go/internal/modcmd/mod.go index 991f4e7..8206971 100644 --- a/vendor/cmd/go/internal/modcmd/mod.go +++ b/vendor/cmd/go/internal/modcmd/mod.go
@@ -84,9 +84,20 @@ type GoMod struct { Module Module - Require []Module + Require []Require Exclude []Module - Replace []struct{ Old, New Module } + Replace []Replace + } + + type Require struct { + Path string + Version string + Indirect bool + } + + type Replace string { + Old Module + New Module } Note that this only describes the go.mod file itself, not other modules @@ -432,11 +443,17 @@ // fileJSON is the -json output data structure. type fileJSON struct { Module module.Version - Require []module.Version + Require []requireJSON Exclude []module.Version Replace []replaceJSON } +type requireJSON struct { + Path string + Version string `json:",omitempty"` + Indirect bool `json:",omitempty"` +} + type replaceJSON struct { Old module.Version New module.Version @@ -449,7 +466,7 @@ var f fileJSON f.Module = modFile.Module.Mod for _, r := range modFile.Require { - f.Require = append(f.Require, r.Mod) + f.Require = append(f.Require, requireJSON{Path: r.Mod.Path, Version: r.Mod.Version, Indirect: r.Indirect}) } for _, x := range modFile.Exclude { f.Exclude = append(f.Exclude, x.Mod)
diff --git a/vendor/cmd/go/internal/modconv/convert.go b/vendor/cmd/go/internal/modconv/convert.go index 6713038..0021478 100644 --- a/vendor/cmd/go/internal/modconv/convert.go +++ b/vendor/cmd/go/internal/modconv/convert.go
@@ -72,7 +72,7 @@ } sort.Strings(paths) for _, path := range paths { - f.AddNewRequire(path, need[path]) + f.AddNewRequire(path, need[path], false) } for _, r := range mf.Replace {
diff --git a/vendor/cmd/go/internal/modfile/rule.go b/vendor/cmd/go/internal/modfile/rule.go index 24bae6b..1b9e7ba 100644 --- a/vendor/cmd/go/internal/modfile/rule.go +++ b/vendor/cmd/go/internal/modfile/rule.go
@@ -36,8 +36,9 @@ // A Require is a single require statement. type Require struct { - Mod module.Version - Syntax *Line + Mod module.Version + Indirect bool // has "// indirect" comment + Syntax *Line } // An Exclude is a single exclude statement. @@ -182,8 +183,9 @@ } if verb == "require" { f.Require = append(f.Require, &Require{ - Mod: module.Version{Path: s, Version: v}, - Syntax: line, + Mod: module.Version{Path: s, Version: v}, + Syntax: line, + Indirect: isIndirect(line), }) } else { f.Exclude = append(f.Exclude, &Exclude{ @@ -253,6 +255,54 @@ } } +// isIndirect reports whether line has a "// indirect" comment, +// meaning it is in go.mod only for its effect on indirect dependencies, +// so that it can be dropped entirely once the effective version of the +// indirect dependency reaches the given minimum version. +func isIndirect(line *Line) bool { + if len(line.Suffix) == 0 { + return false + } + f := strings.Fields(line.Suffix[0].Token) + return (len(f) == 2 && f[1] == "indirect" || len(f) > 2 && f[1] == "indirect;") && f[0] == "//" +} + +// setIndirect sets line to have (or not have) a "// indirect" comment. +func setIndirect(line *Line, indirect bool) { + if isIndirect(line) == indirect { + return + } + if indirect { + // Adding comment. + if len(line.Suffix) == 0 { + // New comment. + line.Suffix = []Comment{{Token: "// indirect", Suffix: true}} + return + } + // Insert at beginning of existing comment. + com := &line.Suffix[0] + space := " " + if len(com.Token) > 2 && com.Token[2] == ' ' || com.Token[2] == '\t' { + space = "" + } + com.Token = "// indirect;" + space + com.Token[2:] + return + } + + // Removing comment. + f := strings.Fields(line.Suffix[0].Token) + if len(f) == 2 { + // Remove whole comment. + line.Suffix = nil + return + } + + // Remove comment prefix. + com := &line.Suffix[0] + i := strings.Index(com.Token, "indirect;") + com.Token = "//" + com.Token[i+len("indirect;"):] +} + // IsDirectoryPath reports whether the given path should be interpreted // as a directory path. Just like on the go command line, relative paths // and rooted paths are directory paths; the rest are module paths. @@ -403,24 +453,29 @@ } if need { - f.AddNewRequire(path, vers) + f.AddNewRequire(path, vers, false) } return nil } -func (f *File) AddNewRequire(path, vers string) { - f.Require = append(f.Require, &Require{module.Version{Path: path, Version: vers}, f.Syntax.addLine(nil, "require", AutoQuote(path), vers)}) +func (f *File) AddNewRequire(path, vers string, indirect bool) { + line := f.Syntax.addLine(nil, "require", AutoQuote(path), vers) + setIndirect(line, indirect) + f.Require = append(f.Require, &Require{module.Version{Path: path, Version: vers}, indirect, line}) } -func (f *File) SetRequire(req []module.Version) { +func (f *File) SetRequire(req []*Require) { need := make(map[string]string) - for _, m := range req { - need[m.Path] = m.Version + indirect := make(map[string]bool) + for _, r := range req { + need[r.Mod.Path] = r.Mod.Version + indirect[r.Mod.Path] = r.Indirect } for _, r := range f.Require { if v, ok := need[r.Mod.Path]; ok { r.Mod.Version = v + r.Indirect = indirect[r.Mod.Path] } } @@ -434,6 +489,7 @@ if p, err := parseString(&line.Token[0]); err == nil && need[p] != "" { line.Token[1] = need[p] delete(need, p) + setIndirect(line, indirect[p]) newLines = append(newLines, line) } } @@ -448,6 +504,7 @@ if p, err := parseString(&stmt.Token[1]); err == nil && need[p] != "" { stmt.Token[2] = need[p] delete(need, p) + setIndirect(stmt, indirect[p]) } else { continue // drop stmt } @@ -458,7 +515,7 @@ f.Syntax.Stmt = newStmts for path, vers := range need { - f.AddNewRequire(path, vers) + f.AddNewRequire(path, vers, indirect[path]) } f.SortBlocks() }
diff --git a/vendor/cmd/go/internal/modinfo/info.go b/vendor/cmd/go/internal/modinfo/info.go index 25deaa5..5a7d6bb 100644 --- a/vendor/cmd/go/internal/modinfo/info.go +++ b/vendor/cmd/go/internal/modinfo/info.go
@@ -10,14 +10,15 @@ // and the fields are documented in the help text in ../list/list.go type ModulePublic struct { - Path string `json:",omitempty"` // module path - Version string `json:",omitempty"` // module version - Replace *ModulePublic `json:",omitempty"` // replaced by this module - Time *time.Time `json:",omitempty"` // time version was created - Update *ModulePublic `json:",omitempty"` // available update (with -u) - Main bool `json:",omitempty"` // is this the main module? - Dir string `json:",omitempty"` // directory holding local copy of files, if any - Error *ModuleError `json:",omitempty"` // error loading module + Path string `json:",omitempty"` // module path + Version string `json:",omitempty"` // module version + Replace *ModulePublic `json:",omitempty"` // replaced by this module + Time *time.Time `json:",omitempty"` // time version was created + Update *ModulePublic `json:",omitempty"` // available update (with -u) + Main bool `json:",omitempty"` // is this the main module? + Indirect bool `json:",omitempty"` // module is only indirectly needed by main module + Dir string `json:",omitempty"` // directory holding local copy of files, if any + Error *ModuleError `json:",omitempty"` // error loading module } type ModuleError struct {
diff --git a/vendor/cmd/go/internal/module/module.go b/vendor/cmd/go/internal/module/module.go index 7fad513..7b32b24 100644 --- a/vendor/cmd/go/internal/module/module.go +++ b/vendor/cmd/go/internal/module/module.go
@@ -26,7 +26,7 @@ // and uses Version = "". // Second, during MVS calculations the version "none" is used // to represent the decision to take no version of a given module. - Version string + Version string `json:",omitempty"` } // Check checks that a given module path, version pair is valid.
diff --git a/vendor/cmd/go/internal/mvs/mvs.go b/vendor/cmd/go/internal/mvs/mvs.go index a16eec0..03303d6 100644 --- a/vendor/cmd/go/internal/mvs/mvs.go +++ b/vendor/cmd/go/internal/mvs/mvs.go
@@ -142,8 +142,9 @@ } // Req returns the minimal requirement list for the target module -// that results in the given build list. -func Req(target module.Version, list []module.Version, reqs Reqs) ([]module.Version, error) { +// that results in the given build list, with the constraint that all +// module paths listed in base must appear in the returned list. +func Req(target module.Version, list []module.Version, base []string, reqs Reqs) ([]module.Version, error) { // Note: Not running in parallel because we assume // that list came from a previous operation that paged // in all the requirements, so there's no I/O to overlap now. @@ -197,7 +198,14 @@ max[m.Path] = m.Version } } + // First walk the base modules that must be listed. var min []module.Version + for _, path := range base { + m := module.Version{Path: path, Version: max[path]} + min = append(min, m) + walk(m) + } + // Now the reverse postorder to bring in anything else. for i := len(postorder) - 1; i >= 0; i-- { m := postorder[i] if max[m.Path] != m.Version {
diff --git a/vendor/cmd/go/internal/mvs/mvs_test.go b/vendor/cmd/go/internal/mvs/mvs_test.go index d52edd5..2a27dfb 100644 --- a/vendor/cmd/go/internal/mvs/mvs_test.go +++ b/vendor/cmd/go/internal/mvs/mvs_test.go
@@ -166,6 +166,13 @@ build A: A B1 C1 D1 upgrade* A: A B2 C2 +name: simplify +A: B1 C1 +B1: C2 +C1: D1 +C2: +build A: A B1 C2 + name: up1 A: B1 C1 B1: @@ -259,6 +266,22 @@ D2: build A: A B1 upgrade* A: A B2 + +# Requirement minimization. + +name: req1 +A: B1 C1 D1 E1 F1 +B1: C1 E1 F1 +req A: B1 D1 +req A C: B1 C1 D1 + +name: req2 +A: G1 H1 +G1: H1 +H1: G1 +req A: G1 +req A G: G1 +req A H: H1 ` func Test(t *testing.T) { @@ -341,19 +364,19 @@ continue case "upgradereq": if len(kf) < 2 { - t.Fatalf("upgrade takes at least one arguments: %q", line) + t.Fatalf("upgrade takes at least one argument: %q", line) } fns = append(fns, func(t *testing.T) { list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) if err == nil { - list, err = Req(m(kf[1]), list, reqs) + list, err = Req(m(kf[1]), list, nil, reqs) } checkList(t, key, list, err, val) }) continue case "upgrade": if len(kf) < 2 { - t.Fatalf("upgrade takes at least one arguments: %q", line) + t.Fatalf("upgrade takes at least one argument: %q", line) } fns = append(fns, func(t *testing.T) { list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) @@ -362,13 +385,26 @@ continue case "downgrade": if len(kf) < 2 { - t.Fatalf("downgrade takes at least one arguments: %q", line) + t.Fatalf("downgrade takes at least one argument: %q", line) } fns = append(fns, func(t *testing.T) { list, err := Downgrade(m(kf[1]), reqs, ms(kf[1:])...) checkList(t, key, list, err, val) }) continue + case "req": + if len(kf) < 2 { + t.Fatalf("req takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := BuildList(m(kf[1]), reqs) + if err != nil { + t.Fatal(err) + } + list, err = Req(m(kf[1]), list, kf[2:], reqs) + checkList(t, key, list, err, val) + }) + continue } if len(kf) == 1 && 'A' <= key[0] && key[0] <= 'Z' { var rs []module.Version
diff --git a/vendor/cmd/go/internal/vgo/build.go b/vendor/cmd/go/internal/vgo/build.go index b06213f..6afe336 100644 --- a/vendor/cmd/go/internal/vgo/build.go +++ b/vendor/cmd/go/internal/vgo/build.go
@@ -41,7 +41,7 @@ if isStandardImportPath(pkgpath) || !Enabled() { return nil } - return moduleInfo(findModule(pkgpath, pkgpath)) + return moduleInfo(findModule(pkgpath, pkgpath), true) } func ModuleInfo(path string) *modinfo.ModulePublic { @@ -50,12 +50,12 @@ } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}) + return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(m) + return moduleInfo(m, true) } } @@ -87,7 +87,7 @@ } } -func moduleInfo(m module.Version) *modinfo.ModulePublic { +func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { return &modinfo.ModulePublic{ Path: m.Path, @@ -97,8 +97,9 @@ } info := &modinfo.ModulePublic{ - Path: m.Path, - Version: m.Version, + Path: m.Path, + Version: m.Version, + Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], } // complete fills in the extra fields in m.
diff --git a/vendor/cmd/go/internal/vgo/init.go b/vendor/cmd/go/internal/vgo/init.go index 4904001..e8868bb 100644 --- a/vendor/cmd/go/internal/vgo/init.go +++ b/vendor/cmd/go/internal/vgo/init.go
@@ -418,11 +418,24 @@ modfetch.WriteGoSum() if buildList != nil { - min, err := mvs.Req(Target, buildList, newReqs(buildList)) + var direct []string + for _, m := range buildList[1:] { + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } + min, err := mvs.Req(Target, buildList, direct, newReqs(buildList)) if err != nil { base.Fatalf("vgo: %v", err) } - modFile.SetRequire(min) + var list []*modfile.Require + for _, m := range min { + list = append(list, &modfile.Require{ + Mod: m, + Indirect: !loaded.direct[m.Path], + }) + } + modFile.SetRequire(list) } file := filepath.Join(ModRoot, "go.mod")
diff --git a/vendor/cmd/go/internal/vgo/list.go b/vendor/cmd/go/internal/vgo/list.go index 5b98d7f..c31e958 100644 --- a/vendor/cmd/go/internal/vgo/list.go +++ b/vendor/cmd/go/internal/vgo/list.go
@@ -17,7 +17,7 @@ func ListModules(args []string) []*modinfo.ModulePublic { LoadBuildList() if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0])} + return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -47,7 +47,7 @@ matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m)) + mods = append(mods, moduleInfo(m, true)) } } }
diff --git a/vendor/cmd/go/internal/vgo/load.go b/vendor/cmd/go/internal/vgo/load.go index 342fda8..0924483 100644 --- a/vendor/cmd/go/internal/vgo/load.go +++ b/vendor/cmd/go/internal/vgo/load.go
@@ -200,6 +200,7 @@ InitMod() loaded = newLoader() + loaded.isALL = true loaded.tags = anyTags loaded.testAll = true all := TargetPackages() @@ -267,6 +268,13 @@ return pkg.mod } +// ModuleUsedDirectly reports whether the main module directly imports +// some package in the module with the given path. +func ModuleUsedDirectly(path string) bool { + return loaded.direct[path] +} + +// Lookup XXX TODO. func Lookup(parentPath, path string) (dir, realPath string, err error) { realPath = ImportMap(path) if realPath == "" { @@ -297,6 +305,7 @@ type loader struct { tags map[string]bool // tags for scanDir testRoots bool // include tests for roots + isALL bool // created with LoadALL testAll bool // include tests for all packages // missingMu protects found, but also buildList, modFile @@ -312,6 +321,9 @@ work *par.Work // current work queue pkgCache *par.Cache // map from string to *loadPkg missing *par.Work // missing work queue + + // computed at end of iterations + direct map[string]bool // imported directly by main module } func newLoader() *loader { @@ -399,6 +411,29 @@ } base.ExitIfErrors() + // Compute directly referenced dependency modules. + ld.direct = make(map[string]bool) + for _, pkg := range ld.pkgs { + if pkg.mod == Target { + for _, dep := range pkg.imports { + if dep.mod.Path != "" { + ld.direct[dep.mod.Path] = true + } + } + } + } + + // Mix in direct markings (really, lack of indirect markings) + // from go.mod, unless we scanned the whole module + // and can therefore be sure we know better than go.mod. + if !ld.isALL && modFile != nil { + for _, r := range modFile.Require { + if !r.Indirect { + ld.direct[r.Mod.Path] = true + } + } + } + buildList = ld.buildList ld.buildList = nil // catch accidental use }
diff --git a/vendor/cmd/go/testdata/vendormod/go.mod b/vendor/cmd/go/testdata/vendormod/go.mod index 801f286..d0ccb06 100644 --- a/vendor/cmd/go/testdata/vendormod/go.mod +++ b/vendor/cmd/go/testdata/vendormod/go.mod
@@ -12,7 +12,7 @@ require ( a v1.0.0 - w v1.0.0 + w v1.0.0 // indirect x v1.0.0 y v1.0.0 z v1.0.0
diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go index e38ee24..070412b 100644 --- a/vendor/cmd/go/vgo_test.go +++ b/vendor/cmd/go/vgo_test.go
@@ -10,6 +10,7 @@ "io/ioutil" "os" "path/filepath" + "regexp" "runtime" "sort" "strings" @@ -163,8 +164,7 @@ tg.run("-vgo", "mod", "-json") want := `{ "Module": { - "Path": "x.x/y/z", - "Version": "" + "Path": "x.x/y/z" }, "Require": [ { @@ -185,8 +185,7 @@ "Version": "v1.4.0" }, "New": { - "Path": "../z", - "Version": "" + "Path": "../z" } } ] @@ -392,8 +391,45 @@ `), 0666)) tg.run("-vgo", "get", "-x", "-u") - tg.run("-vgo", "list", "-m", "all") + tg.run("-vgo", "list", "-m", "-f={{.Path}} {{.Version}}{{if .Indirect}} // indirect{{end}}", "all") tg.grepStdout(`quote v1.5.2$`, "should have upgraded only to v1.5.2") + tg.grepStdout(`x/text [v0-9a-f.\-]+ // indirect`, "should list golang.org/x/text as indirect") + + var gomod string + readGoMod := func() { + data, err := ioutil.ReadFile(tg.path("x/go.mod")) + if err != nil { + t.Fatal(err) + } + gomod = string(data) + } + readGoMod() + if !strings.Contains(gomod, "rsc.io/quote v1.5.2\n") { + t.Fatalf("expected rsc.io/quote direct requirement:\n%s", gomod) + } + if !regexp.MustCompile(`(?m)golang.org/x/text.* // indirect`).MatchString(gomod) { + t.Fatalf("expected golang.org/x/text indirect requirement:\n%s", gomod) + } + + tg.must(ioutil.WriteFile(tg.path("x/x.go"), []byte(`package x; import _ "golang.org/x/text"`), 0666)) + tg.run("-vgo", "list") // rescans directory + readGoMod() + if !strings.Contains(gomod, "rsc.io/quote v1.5.2\n") { + t.Fatalf("expected rsc.io/quote direct requirement:\n%s", gomod) + } + if !regexp.MustCompile(`(?m)golang.org/x/text[^/]+\n`).MatchString(gomod) { + t.Fatalf("expected golang.org/x/text DIRECT requirement:\n%s", gomod) + } + + tg.must(ioutil.WriteFile(tg.path("x/x.go"), []byte(`package x; import _ "rsc.io/quote"`), 0666)) + tg.run("-vgo", "mod", "-sync") // rescans everything, can put // indirect marks back + readGoMod() + if !strings.Contains(gomod, "rsc.io/quote v1.5.2\n") { + t.Fatalf("expected rsc.io/quote direct requirement:\n%s", gomod) + } + if !regexp.MustCompile(`(?m)golang.org/x/text.* // indirect\n`).MatchString(gomod) { + t.Fatalf("expected golang.org/x/text indirect requirement:\n%s", gomod) + } tg.run("-vgo", "get", "-m", "rsc.io/quote@dd9747d") tg.run("-vgo", "list", "-m", "all")