cmd/go/internal/modload: finish Import implementation
modload.Import now means "figure out what this import means in the main module",
including searching for a module to add to go.mod if the import is missing.
(Previously it only did the search.)
Fix Import to handle overlapping module file trees correctly:
the fact that x/y/z is in the build list does not mean it is the
module that must provide x/y/z/v2/w or even x/y/z/w.
The definition of "go get x@v" is now easy to explain: if x is a module path,
go get is talking about that module. Otherwise go get is talking about the
module that provides or would provide x if encountered as an import in the
main module.
Simplify the package load process by using the new Import to eliminate
the separate doMissing parallel pass, as suggested by Bryan in an
earlier code review.
Drop modload.SrcMod in favor of modfetch.SrcMod,
and guard against SrcMod unset (filepath.Join(x, y) treats x="" as like x="."),
which can happen in tests.
Fixes golang/go#24851.
Fixes golang/go#26048.
Fixes golang/go#26250.
Change-Id: I16abfa0c095492fc10ccd75f5857c1c087935867
Reviewed-on: https://go-review.googlesource.com/123095
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/vendor/cmd/go/internal/load/pkg.go b/vendor/cmd/go/internal/load/pkg.go
index edc29ad..9f28fd7 100644
--- a/vendor/cmd/go/internal/load/pkg.go
+++ b/vendor/cmd/go/internal/load/pkg.go
@@ -513,6 +513,9 @@
} else if modErr != nil {
bp = new(build.Package)
err = fmt.Errorf("unknown import path %q: %v", importPath, modErr)
+ } else if cfg.ModulesEnabled && path != "unsafe" {
+ bp = new(build.Package)
+ err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", importPath)
} else {
buildMode := build.ImportComment
if mode&ResolveImport == 0 || path != origPath {
diff --git a/vendor/cmd/go/internal/modcmd/verify.go b/vendor/cmd/go/internal/modcmd/verify.go
index 72bd7a5..27cd9ed 100644
--- a/vendor/cmd/go/internal/modcmd/verify.go
+++ b/vendor/cmd/go/internal/modcmd/verify.go
@@ -13,6 +13,7 @@
"cmd/go/internal/base"
"cmd/go/internal/dirhash"
+ "cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/module"
)
@@ -29,9 +30,9 @@
func verifyMod(mod module.Version) bool {
ok := true
- zip := filepath.Join(modload.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
+ zip := filepath.Join(modfetch.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
_, zipErr := os.Stat(zip)
- dir := filepath.Join(modload.SrcMod, mod.Path+"@"+mod.Version)
+ dir := filepath.Join(modfetch.SrcMod, mod.Path+"@"+mod.Version)
_, dirErr := os.Stat(dir)
data, err := ioutil.ReadFile(zip + "hash")
if err != nil {
diff --git a/vendor/cmd/go/internal/modfetch/cache.go b/vendor/cmd/go/internal/modfetch/cache.go
index 5503df1..587b2a6 100644
--- a/vendor/cmd/go/internal/modfetch/cache.go
+++ b/vendor/cmd/go/internal/modfetch/cache.go
@@ -236,6 +236,11 @@
// just to find out about a commit we already know about
// (and have cached under its pseudo-version).
func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) {
+ if SrcMod == "" {
+ // Do not download to current directory.
+ return "", nil, errNotCached
+ }
+
if !codehost.AllHex(rev) || len(rev) < 12 {
return "", nil, errNotCached
}
diff --git a/vendor/cmd/go/internal/modfetch/fetch.go b/vendor/cmd/go/internal/modfetch/fetch.go
index aff2cc8..87797f9 100644
--- a/vendor/cmd/go/internal/modfetch/fetch.go
+++ b/vendor/cmd/go/internal/modfetch/fetch.go
@@ -28,6 +28,11 @@
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
func Download(mod module.Version) (dir string, err error) {
+ if SrcMod == "" {
+ // Do not download to current directory.
+ return "", fmt.Errorf("missing modfetch.SrcMod")
+ }
+
// The par.Cache here avoids duplicate work but also
// avoids conflicts from simultaneous calls by multiple goroutines
// for the same version.
@@ -190,6 +195,11 @@
// checkSum checks the given module's checksum.
func checkSum(mod module.Version) {
+ if SrcMod == "" {
+ // Do not use current directory.
+ return
+ }
+
// Do the file I/O before acquiring the go.sum lock.
data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
if err != nil {
@@ -245,6 +255,11 @@
// Sum returns the checksum for the downloaded copy of the given module,
// if present in the download cache.
func Sum(mod module.Version) string {
+ if SrcMod == "" {
+ // Do not use current directory.
+ return ""
+ }
+
data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
if err != nil {
return ""
diff --git a/vendor/cmd/go/internal/modget/get.go b/vendor/cmd/go/internal/modget/get.go
index b8713d1..4f97dbb 100644
--- a/vendor/cmd/go/internal/modget/get.go
+++ b/vendor/cmd/go/internal/modget/get.go
@@ -511,24 +511,22 @@
return module.Version{}, err
}
- // Otherwise fall back to resolving package path as of today
- // and applying the version to the resulting module.
- // We could do more subtle things when older versions are
- // specified, but this seems good enough and more predictable.
- // If this behavior is wrong, the user can always specify the
- // desired module path instead of a package path,
- // and then the code above will handle it.
- repo, info, err := modload.Import(path, modload.Allowed)
+ // Otherwise, interpret the package path as an import
+ // and determine what module that import would address
+ // if found in the current source code.
+ // Then apply the version to that module.
+ m, _, err := modload.Import(path)
if err != nil {
return module.Version{}, err
}
- if vers != "latest" {
- // modload.Import returned "latest" version. Look up requested version.
- if info, err = modload.Query(repo.ModulePath(), vers, modload.Allowed); err != nil {
- return module.Version{}, err
- }
+ if m.Path == "" {
+ return module.Version{}, fmt.Errorf("package %q is not in a module", path)
}
- return module.Version{Path: repo.ModulePath(), Version: info.Version}, nil
+ info, err = modload.Query(m.Path, vers, modload.Allowed)
+ if err != nil {
+ return module.Version{}, err
+ }
+ return module.Version{Path: m.Path, Version: info.Version}, nil
}
// isModulePath reports whether path names an actual module,
diff --git a/vendor/cmd/go/internal/modload/build.go b/vendor/cmd/go/internal/modload/build.go
index 17cbcda..13424a2 100644
--- a/vendor/cmd/go/internal/modload/build.go
+++ b/vendor/cmd/go/internal/modload/build.go
@@ -117,7 +117,7 @@
}
if semver.IsValid(m.Version) {
- dir := filepath.Join(SrcMod, m.Path+"@"+m.Version)
+ dir := filepath.Join(modfetch.SrcMod, m.Path+"@"+m.Version)
if stat, err := os.Stat(dir); err == nil && stat.IsDir() {
m.Dir = dir
}
@@ -194,11 +194,12 @@
}
func findModule(target, path string) module.Version {
+ // TODO: This should use loaded.
if path == "." {
return buildList[0]
}
for _, mod := range buildList {
- if importPathInModule(path, mod.Path) {
+ if maybeInModule(path, mod.Path) {
return mod
}
}
diff --git a/vendor/cmd/go/internal/modload/import.go b/vendor/cmd/go/internal/modload/import.go
index 3956f75..e9dff9f 100644
--- a/vendor/cmd/go/internal/modload/import.go
+++ b/vendor/cmd/go/internal/modload/import.go
@@ -5,62 +5,263 @@
package modload
import (
+ "bytes"
+ "errors"
"fmt"
+ "go/build"
+ "os"
pathpkg "path"
+ "path/filepath"
+ "strings"
"cmd/go/internal/cfg"
- "cmd/go/internal/modfetch"
"cmd/go/internal/module"
+ "cmd/go/internal/par"
+ "cmd/go/internal/search"
)
-// Import returns the module repo and version to use to satisfy the given import path.
-// It considers a sequence of module paths starting with the import path and
-// removing successive path elements from the end. It stops when it finds a module
-// path for which the latest version of the module provides the expected package.
-// If non-nil, the allowed function is used to filter known versions of a given module
-// before determining which one is "latest".
-func Import(path string, allowed func(module.Version) bool) (modfetch.Repo, *modfetch.RevInfo, error) {
- if cfg.BuildGetmode != "" {
- return nil, nil, fmt.Errorf("import resolution disabled by -getmode=%s", cfg.BuildGetmode)
+type ImportMissingError struct {
+ ImportPath string
+ Module module.Version
+}
+
+func (e *ImportMissingError) Error() string {
+ if e.Module.Path == "" {
+ return "cannot find module providing package " + e.ImportPath
+ }
+ return "missing module for import: " + e.Module.Path + "@" + e.Module.Version + " provides " + e.ImportPath
+}
+
+// Import finds the module and directory in the build list
+// containing the package with the given import path.
+// The answer must be unique: Import returns an error
+// if multiple modules attempt to provide the same package.
+// Import can return a module with an empty m.Path, for packages in the standard library.
+// Import can return an empty directory string, for fake packages like "C" and "unsafe".
+//
+// If the package cannot be found in the current build list,
+// Import returns an ImportMissingError as the error.
+// If Import can identify a module that could be added to supply the package,
+// the ImportMissingErr records that module.
+func Import(path string) (m module.Version, dir string, err error) {
+ if strings.Contains(path, "@") {
+ return module.Version{}, "", fmt.Errorf("import path should not have @version")
+ }
+ if build.IsLocalImport(path) {
+ return module.Version{}, "", fmt.Errorf("relative import not supported")
+ }
+ if path == "C" || path == "unsafe" {
+ // There's no directory for import "C" or import "unsafe".
+ return module.Version{}, "", nil
}
- try := func(path string) (modfetch.Repo, *modfetch.RevInfo, error) {
- r, err := modfetch.Lookup(path)
- if err != nil {
- return nil, nil, err
+ // Is the package in the standard library?
+ if search.IsStandardImportPath(path) {
+ if strings.HasPrefix(path, "golang_org/") {
+ return module.Version{}, filepath.Join(cfg.GOROOT, "src/vendor", path), nil
}
- info, err := Query(path, "latest", allowed)
- if err != nil {
- return nil, nil, err
+ dir := filepath.Join(cfg.GOROOT, "src", path)
+ if _, err := os.Stat(dir); err == nil {
+ return module.Version{}, dir, nil
}
- _, err = r.GoMod(info.Version)
- if err != nil {
- return nil, nil, err
- }
- // TODO(rsc): Do what the docs promise: download the module
- // source code and check that it actually contains code for the
- // target import path. To do that efficiently we will need to move
- // the unzipped code cache out of ../modload into this package.
- // TODO(rsc): When this happens, look carefully at the use of
- // modfetch.Import in modget.getQuery.
- return r, info, nil
}
- // Find enclosing module by walking up path element by element.
- var firstErr error
- for {
- r, info, err := try(path)
- if err == nil {
- return r, info, nil
+ // -getmode=vendor is special.
+ // Everything must be in the main module or the main module's vendor directory.
+ if cfg.BuildGetmode == "vendor" {
+ mainDir, mainOK := dirInModule(path, Target.Path, ModRoot, true)
+ vendorDir, vendorOK := dirInModule(path, "", filepath.Join(ModRoot, "vendor"), false)
+ if mainOK && vendorOK {
+ return module.Version{}, "", fmt.Errorf("ambiguous import: found %s in multiple directories:\n\t%s\n\t%s", path, mainDir, vendorDir)
}
- if firstErr == nil {
- firstErr = err
+ // Prefer to return main directory if there is one,
+ // Note that we're not checking that the package exists.
+ // We'll leave that for load.
+ if !vendorOK && mainDir != "" {
+ return Target, mainDir, nil
}
- p := pathpkg.Dir(path)
- if p == "." {
- break
- }
- path = p
+ readVendorList()
+ return vendorMap[path], vendorDir, nil
}
- return nil, nil, firstErr
+
+ // Check each module on the build list.
+ var dirs []string
+ var mods []module.Version
+ for _, m := range buildList {
+ if !maybeInModule(path, m.Path) {
+ // Avoid possibly downloading irrelevant modules.
+ continue
+ }
+ root, isLocal, err := fetch(m)
+ if err != nil {
+ // Report fetch error.
+ // Note that we don't know for sure this module is necessary,
+ // but it certainly _could_ provide the package, and even if we
+ // continue the loop and find the package in some other module,
+ // we need to look at this module to make sure the import is
+ // not ambiguous.
+ return module.Version{}, "", err
+ }
+ dir, ok := dirInModule(path, m.Path, root, isLocal)
+ if ok {
+ mods = append(mods, m)
+ dirs = append(dirs, dir)
+ }
+ }
+ if len(mods) == 1 {
+ return mods[0], dirs[0], nil
+ }
+ if len(mods) > 0 {
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "ambiguous import: found %s in multiple modules:", path)
+ for i, m := range mods {
+ fmt.Fprintf(&buf, "\n\t%s", m.Path)
+ if m.Version != "" {
+ fmt.Fprintf(&buf, " %s", m.Version)
+ }
+ fmt.Fprintf(&buf, " (%s)", dirs[i])
+ }
+ return module.Version{}, "", errors.New(buf.String())
+ }
+
+ // Special case: if the path matches a module path,
+ // and we haven't found code in any module on the build list
+ // (since we haven't returned yet),
+ // force the use of the current module instead of
+ // looking for an alternate one.
+ // This helps "go get golang.org/x/net" even though
+ // there is no code in x/net.
+ for _, m := range buildList {
+ if m.Path == path {
+ root, isLocal, err := fetch(m)
+ if err != nil {
+ return module.Version{}, "", err
+ }
+ dir, _ := dirInModule(path, m.Path, root, isLocal)
+ return m, dir, nil
+ }
+ }
+
+ // Not on build list.
+
+ // Look up module containing the package, for addition to the build list.
+ // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing.
+ if cfg.BuildGetmode == "local" {
+ return module.Version{}, "", fmt.Errorf("import lookup disabled by -getmode=local")
+ }
+
+ for p := path; p != "."; p = pathpkg.Dir(p) {
+ // We can't upgrade the main module.
+ // Note that this loop does consider upgrading other modules on the build list.
+ // If that's too aggressive we can skip all paths already on the build list,
+ // not just Target.Path, but for now let's try being aggressive.
+ if p == Target.Path {
+ // Can't move to a new version of main module.
+ continue
+ }
+
+ info, err := Query(p, "latest", Allowed)
+ if err != nil {
+ continue
+ }
+ m := module.Version{Path: p, Version: info.Version}
+ root, isLocal, err := fetch(m)
+ if err != nil {
+ continue
+ }
+ _, ok := dirInModule(path, m.Path, root, isLocal)
+ if ok {
+ return module.Version{}, "", &ImportMissingError{ImportPath: path, Module: m}
+ }
+
+ // Special case matching the one above:
+ // if m.Path matches path, assume adding it to the build list
+ // will either add the right code or the right code doesn't exist.
+ if m.Path == path {
+ return module.Version{}, "", &ImportMissingError{ImportPath: path, Module: m}
+ }
+ }
+
+ // Did not resolve import to any module.
+ // TODO(rsc): It would be nice to return a specific error encountered
+ // during the loop above if possible, but it's not clear how to pick
+ // out the right one.
+ return module.Version{}, "", &ImportMissingError{ImportPath: path}
+}
+
+// maybeInModule reports whether, syntactically,
+// a package with the given import path could be supplied
+// by a module with the given module path (mpath).
+func maybeInModule(path, mpath string) bool {
+ return mpath == path ||
+ len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath
+}
+
+var haveGoModCache, haveGoFilesCache par.Cache
+
+// dirInModule locates the directory that would hold the package named by the given path,
+// if it were in the module with module path mpath and root mdir.
+// If path is syntactically not within mpath,
+// or if mdir is a local file tree (isLocal == true) and the directory
+// that would hold path is in a sub-module (covered by a go.mod below mdir),
+// dirInModule returns "", false.
+//
+// Otherwise, dirInModule returns the name of the directory where
+// Go source files would be expected, along with a boolean indicating
+// whether there are in fact Go source files in that directory.
+func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool) {
+ // Determine where to expect the package.
+ if path == mpath {
+ dir = mdir
+ } else if mpath == "" { // vendor directory
+ dir = filepath.Join(mdir, path)
+ } else if len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath {
+ dir = filepath.Join(mdir, path[len(mpath)+1:])
+ } else {
+ return "", false
+ }
+
+ // Check that there aren't other modules in the way.
+ // This check is unnecessary inside the module cache
+ // and important to skip in the vendor directory,
+ // where all the module trees have been overlaid.
+ // So we only check local module trees
+ // (the main module, and any directory trees pointed at by replace directives).
+ if isLocal {
+ for d := dir; d != mdir && len(d) > len(mdir); d = filepath.Dir(d) {
+ haveGoMod := haveGoModCache.Do(d, func() interface{} {
+ _, err := os.Stat(filepath.Join(d, "go.mod"))
+ return err == nil
+ }).(bool)
+
+ if haveGoMod {
+ return "", false
+ }
+ }
+ }
+
+ // Now committed to returning dir (not "").
+
+ // Are there Go source files in the directory?
+ // We don't care about build tags, not even "+build ignore".
+ // We're just looking for a plausible directory.
+ haveGoFiles = haveGoFilesCache.Do(dir, func() interface{} {
+ f, err := os.Open(dir)
+ if err != nil {
+ return false
+ }
+ defer f.Close()
+ names, _ := f.Readdirnames(-1)
+ for _, name := range names {
+ if strings.HasSuffix(name, ".go") {
+ info, err := os.Stat(filepath.Join(dir, name))
+ if err == nil && info.Mode().IsRegular() {
+ return true
+ }
+ }
+ }
+ return false
+ }).(bool)
+
+ return dir, haveGoFiles
}
diff --git a/vendor/cmd/go/internal/modload/import_test.go b/vendor/cmd/go/internal/modload/import_test.go
index a7ca749..8e01dc5 100644
--- a/vendor/cmd/go/internal/modload/import_test.go
+++ b/vendor/cmd/go/internal/modload/import_test.go
@@ -6,35 +6,38 @@
import (
"internal/testenv"
+ "regexp"
"strings"
"testing"
)
var importTests = []struct {
- path string
- mpath string
- err string
+ path string
+ err string
}{
{
- path: "golang.org/x/net/context",
- mpath: "golang.org/x/net",
+ path: "golang.org/x/net/context",
+ err: "missing module for import: golang.org/x/net@.* provides golang.org/x/net/context",
},
{
- path: "github.com/rsc/quote/buggy",
- mpath: "github.com/rsc/quote",
+ path: "golang.org/x/net",
+ err: "missing module for import: golang.org/x/net@.* provides golang.org/x/net",
},
{
- path: "golang.org/x/net",
- mpath: "golang.org/x/net",
+ path: "golang.org/x/text",
+ err: "missing module for import: golang.org/x/text@.* provides golang.org/x/text",
},
{
- path: "github.com/rsc/quote",
- mpath: "github.com/rsc/quote",
+ path: "github.com/rsc/quote/buggy",
+ err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote/buggy",
+ },
+ {
+ path: "github.com/rsc/quote",
+ err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote",
},
{
path: "golang.org/x/foo/bar",
- // TODO(rsc): This error comes from old go get and is terrible. Fix.
- err: `unrecognized import path "golang.org/x/foo/bar" (parse https://golang.org/x/foo/bar?go-get=1: no go-import meta tags ())`,
+ err: "cannot find module providing package golang.org/x/foo/bar",
},
}
@@ -43,18 +46,13 @@
for _, tt := range importTests {
t.Run(strings.Replace(tt.path, "/", "_", -1), func(t *testing.T) {
- repo, info, err := Import(tt.path, nil)
- if tt.err != "" {
- if err != nil && err.Error() == tt.err {
- return
- }
- t.Fatalf("Import(%q): %v, want error %q", tt.path, err, tt.err)
+ // Note that there is no build list, so Import should always fail.
+ m, dir, err := Import(tt.path)
+ if err == nil {
+ t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir)
}
- if err != nil {
- t.Fatalf("Import(%q): %v", tt.path, err)
- }
- if mpath := repo.ModulePath(); mpath != tt.mpath {
- t.Errorf("repo.ModulePath() = %q (%v), want %q", mpath, info.Version, tt.mpath)
+ if !regexp.MustCompile(tt.err).MatchString(err.Error()) {
+ t.Fatalf("Import(%q): error %q, want error matching %#q", tt.path, err, tt.err)
}
})
}
diff --git a/vendor/cmd/go/internal/modload/init.go b/vendor/cmd/go/internal/modload/init.go
index 12a9e3d..a360a2c 100644
--- a/vendor/cmd/go/internal/modload/init.go
+++ b/vendor/cmd/go/internal/modload/init.go
@@ -42,7 +42,6 @@
Target module.Version
gopath string
- SrcMod string // GOPATH/src/mod directory where versioned cache lives
CmdModInit bool // go mod -init flag
CmdModModule string // go mod -module flag
@@ -209,16 +208,16 @@
}
srcV := filepath.Join(list[0], "src/v")
- SrcMod = filepath.Join(list[0], "src/mod")
+ srcMod := filepath.Join(list[0], "src/mod")
infoV, errV := os.Stat(srcV)
- _, errMod := os.Stat(SrcMod)
+ _, errMod := os.Stat(srcMod)
if errV == nil && infoV.IsDir() && errMod != nil && os.IsNotExist(errMod) {
- os.Rename(srcV, SrcMod)
+ os.Rename(srcV, srcMod)
}
- modfetch.SrcMod = SrcMod
+ modfetch.SrcMod = srcMod
modfetch.GoSumFile = filepath.Join(ModRoot, "go.sum")
- codehost.WorkRoot = filepath.Join(SrcMod, "cache/vcs")
+ codehost.WorkRoot = filepath.Join(srcMod, "cache/vcs")
if CmdModInit {
// Running go mod -init: do legacy module conversion
diff --git a/vendor/cmd/go/internal/modload/load.go b/vendor/cmd/go/internal/modload/load.go
index 3f67fc4..2528db7 100644
--- a/vendor/cmd/go/internal/modload/load.go
+++ b/vendor/cmd/go/internal/modload/load.go
@@ -332,16 +332,11 @@
isALL bool // created with LoadALL
testAll bool // include tests for all packages
- // missingMu protects found, but also buildList, modFile
- missingMu sync.Mutex
- found map[string]bool
-
// reset on each iteration
roots []*loadPkg
pkgs []*loadPkg
work *par.Work // current work queue
pkgCache *par.Cache // map from string to *loadPkg
- missing *par.Work // missing work queue
// computed at end of iterations
direct map[string]bool // imported directly by main module
@@ -350,7 +345,6 @@
func newLoader() *loader {
ld := new(loader)
ld.tags = imports.Tags()
- ld.found = make(map[string]bool)
switch cfg.CmdName {
case "test", "vet":
@@ -364,7 +358,6 @@
ld.pkgs = nil
ld.work = new(par.Work)
ld.pkgCache = new(par.Cache)
- ld.missing = nil
}
// A loadPkg records information about a single loaded package.
@@ -392,6 +385,7 @@
base.Fatalf("go: %v", err)
}
+ added := make(map[string]bool)
for {
ld.reset()
if roots != nil {
@@ -404,22 +398,34 @@
}
ld.work.Do(10, ld.doPkg)
ld.buildStacks()
+ numAdded := 0
+ haveMod := make(map[module.Version]bool)
+ for _, m := range buildList {
+ haveMod[m] = true
+ }
for _, pkg := range ld.pkgs {
- if pkg.err == errMissing {
- if ld.missing == nil {
- ld.missing = new(par.Work)
+ if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" {
+ if added[pkg.path] {
+ base.Fatalf("go: %s: looping trying to add package", pkg.stackText())
}
- ld.missing.Add(pkg)
- } else if pkg.err != nil {
+ added[pkg.path] = true
+ numAdded++
+ if !haveMod[err.Module] {
+ haveMod[err.Module] = true
+ buildList = append(buildList, err.Module)
+ }
+ continue
+ }
+ if pkg.err != nil {
base.Errorf("go: %s: %s", pkg.stackText(), pkg.err)
}
}
- if ld.missing == nil {
+ base.ExitIfErrors()
+ if numAdded == 0 {
break
}
- ld.missing.Do(10, ld.findMissing)
- base.ExitIfErrors()
+ // Recompute buildList with all our additions.
buildList, err = mvs.BuildList(Target, Reqs())
if err != nil {
base.Fatalf("go: %v", err)
@@ -449,6 +455,9 @@
}
}
}
+
+ // Check for visibility violations.
+ // TODO!
}
// pkg returns the *loadPkg for path, creating and queuing it if needed.
@@ -485,7 +494,7 @@
pkg.mod = pkg.testOf.mod
imports = pkg.testOf.testImports
} else {
- pkg.dir, pkg.mod, pkg.err = ld.findDir(pkg.path)
+ pkg.mod, pkg.dir, pkg.err = ld.doImport(pkg.path)
if pkg.dir == "" {
return
}
@@ -517,152 +526,24 @@
}
}
-// importPathInModule reports whether, syntactically,
-// a package with the given import path could be supplied
-// by a module with the given module path (mpath).
-func importPathInModule(path, mpath string) bool {
- return mpath == path ||
- len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath
-}
-
-// findDir finds the directory holding source code for the given import path.
-// It returns the directory, the module containing the directory,
+// doImport finds the directory holding source code for the given import path.
+// It returns the module containing the package (if any),
+// the directory containing the package (if any),
// and any error encountered.
-// It is possible to return successfully (err == nil) with an empty directory,
-// for built-in packages like "unsafe" and "C".
-// It is also possible to return successfully with a zero module.Version,
-// for packages in the standard library or when using vendored code.
-func (ld *loader) findDir(path string) (dir string, mod module.Version, err error) {
+// Not all packages have modules: the ones in the standard library do not.
+// Not all packages have directories: "unsafe" and "C" do not.
+func (ld *loader) doImport(path string) (mod module.Version, dir string, err error) {
if strings.Contains(path, "@") {
// Leave for error during load.
- return
+ return module.Version{}, "", nil
}
-
- // Is the package in the standard library?
- if search.IsStandardImportPath(path) {
- if path == "C" || path == "unsafe" {
- // There's no directory for import "C" or import "unsafe".
- return "", module.Version{}, nil
- }
- if strings.HasPrefix(path, "golang_org/") {
- return filepath.Join(cfg.GOROOT, "src/vendor", path), module.Version{}, nil
- }
- dir := filepath.Join(cfg.GOROOT, "src", path)
- if _, err := os.Stat(dir); err == nil {
- return dir, module.Version{}, nil
- }
- }
-
- // Is the package in the main module?
- // Note that having the main module path as a prefix
- // does not guarantee that the package is in the
- // main module. It might still be supplied by some
- // other module. For example, this might be
- // module x/y, and we might be looking for x/y/v2/z.
- // or maybe x/y/z/w in separate module x/y/z.
- var mainDir string
- if importPathInModule(path, Target.Path) {
- mainDir = ModRoot
- if len(path) > len(Target.Path) {
- mainDir = filepath.Join(ModRoot, path[len(Target.Path)+1:])
- }
- if _, err := os.Stat(mainDir); err == nil {
- return mainDir, Target, nil
- }
- }
-
- // With -getmode=vendor, we expect everything else to be in vendor.
- if cfg.BuildGetmode == "vendor" {
- // Using -getmode=vendor, everything the module needs
- // (beyond the current module and standard library)
- // must be in the module's vendor directory.
- // If the package exists in vendor, use it.
- // If the package is not covered by the main module (mainDir == ""), use vendor.
- // Otherwise, if the package could be in either place but is in neither, report the main module.
- vendorDir := filepath.Join(ModRoot, "vendor", path)
- if _, err := os.Stat(vendorDir); err == nil || mainDir == "" {
- // TODO(rsc): We could look up the module information from vendor/modules.txt.
- return vendorDir, module.Version{}, nil
- }
- return mainDir, Target, nil
- }
-
- // Scan all the possible modules that might contain this package,
- // and complain if there are multiple choices. This correctly handles
- // module boundaries that change over time, detecting mismatched
- // module version pairings.
- // (See comment about module paths in modfetch/repo.go.)
- var mod1 module.Version
- var dir1 string
- for _, mod := range buildList {
- if !importPathInModule(path, mod.Path) {
- continue
- }
- dir, err := fetch(mod)
- if err != nil {
- return "", module.Version{}, err
- }
- if len(path) > len(mod.Path) {
- dir = filepath.Join(dir, path[len(mod.Path)+1:])
- }
- if dir1 != "" {
- return "", module.Version{}, fmt.Errorf("found in both %v@%v and %v@%v", mod1.Path, mod1.Version, mod.Path, mod.Version)
- }
- dir1 = dir
- mod1 = mod
- }
- if dir1 != "" {
- return dir1, mod1, nil
- }
- return "", module.Version{}, errMissing
-}
-
-func (ld *loader) findMissing(item interface{}) {
- pkg := item.(*loadPkg)
- path := pkg.path
if build.IsLocalImport(path) {
- base.Errorf("go: relative import is not supported: %s", path)
+ // Leave for error during load.
+ // (Module mode does not allow local imports.)
+ return module.Version{}, "", nil
}
- // TODO: This is wrong (if path = foo/v2/bar and m.Path is foo,
- // maybe we should fall through to the loop at the bottom and check foo/v2).
- ld.missingMu.Lock()
- for _, m := range buildList {
- if importPathInModule(path, m.Path) {
- ld.missingMu.Unlock()
- return
- }
- }
- ld.missingMu.Unlock()
-
- fmt.Fprintf(os.Stderr, "resolving import %q\n", path)
- repo, info, err := Import(path, Allowed)
- if err != nil {
- base.Errorf("go: %s: %v", pkg.stackText(), err)
- return
- }
-
- root := repo.ModulePath()
-
- ld.missingMu.Lock()
- defer ld.missingMu.Unlock()
-
- // Double-check before adding repo twice.
- for _, m := range buildList {
- if importPathInModule(path, m.Path) {
- return
- }
- }
-
- fmt.Fprintf(os.Stderr, "go: finding %s (latest)\n", root)
-
- if ld.found[path] {
- base.Fatalf("internal error: findMissing loop on %s", path)
- }
- ld.found[path] = true
- fmt.Fprintf(os.Stderr, "go: adding %s %s\n", root, info.Version)
- buildList = append(buildList, module.Version{Path: root, Version: info.Version})
- modFile.AddRequire(root, info.Version)
+ return Import(path)
}
// scanDir is like imports.ScanDir but elides known magic imports from the list,
@@ -759,6 +640,10 @@
// If there is no replacement for mod, Replacement returns
// a module.Version with Path == "".
func Replacement(mod module.Version) module.Version {
+ if modFile == nil {
+ // Happens during testing.
+ return module.Version{}
+ }
var found *modfile.Replace
for _, r := range modFile.Replace {
if r.Old == mod {
@@ -820,21 +705,35 @@
}
var vendorOnce sync.Once
-var vendorList []module.Version
+
+var (
+ vendorList []module.Version
+ vendorMap map[string]module.Version
+)
// readVendorList reads the list of vendored modules from vendor/modules.txt.
func readVendorList() {
- var list []module.Version
- data, _ := ioutil.ReadFile(filepath.Join(ModRoot, "vendor/modules.txt"))
- for _, line := range strings.Split(string(data), "\n") {
- if strings.HasPrefix(line, "# ") {
- f := strings.Fields(line)
- if len(f) == 3 && semver.IsValid(f[2]) {
- list = append(list, module.Version{Path: f[1], Version: f[2]})
+ vendorOnce.Do(func() {
+ vendorList = nil
+ vendorMap = make(map[string]module.Version)
+ data, _ := ioutil.ReadFile(filepath.Join(ModRoot, "vendor/modules.txt"))
+ var m module.Version
+ for _, line := range strings.Split(string(data), "\n") {
+ if strings.HasPrefix(line, "# ") {
+ f := strings.Fields(line)
+ m = module.Version{}
+ if len(f) == 3 && semver.IsValid(f[2]) {
+ m = module.Version{Path: f[1], Version: f[2]}
+ vendorList = append(vendorList, m)
+ }
+ } else if m.Path != "" {
+ f := strings.Fields(line)
+ if len(f) == 1 {
+ vendorMap[f[0]] = m
+ }
}
}
- }
- vendorList = list
+ })
}
func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) {
@@ -847,8 +746,7 @@
if cfg.BuildGetmode == "vendor" {
// For every module other than the target,
// return the full list of modules from modules.txt.
- // But only read vendor/modules.txt once.
- vendorOnce.Do(readVendorList)
+ readVendorList()
return vendorList, nil
}
@@ -973,17 +871,21 @@
return module.Version{Path: m.Path, Version: "none"}, nil
}
-func fetch(mod module.Version) (dir string, err error) {
+func fetch(mod module.Version) (dir string, isLocal bool, err error) {
+ if mod == Target {
+ return ModRoot, true, nil
+ }
if r := Replacement(mod); r.Path != "" {
if r.Version == "" {
dir = r.Path
if !filepath.IsAbs(dir) {
dir = filepath.Join(ModRoot, dir)
}
- return dir, nil
+ return dir, true, nil
}
mod = r
}
- return modfetch.Download(mod)
+ dir, err = modfetch.Download(mod)
+ return dir, false, err
}
diff --git a/vendor/cmd/go/internal/modload/query_test.go b/vendor/cmd/go/internal/modload/query_test.go
index 2f84c4f..eb194b5 100644
--- a/vendor/cmd/go/internal/modload/query_test.go
+++ b/vendor/cmd/go/internal/modload/query_test.go
@@ -10,9 +10,11 @@
"log"
"os"
"path"
+ "path/filepath"
"strings"
"testing"
+ "cmd/go/internal/modfetch"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
)
@@ -22,13 +24,13 @@
}
func testMain(m *testing.M) int {
- dir, err := ioutil.TempDir("", "gitrepo-test-")
+ dir, err := ioutil.TempDir("", "modload-test-")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
-
- codehost.WorkRoot = dir
+ modfetch.SrcMod = filepath.Join(dir, "src/mod")
+ codehost.WorkRoot = filepath.Join(dir, "codework")
return m.Run()
}
diff --git a/vendor/cmd/go/internal/modload/search.go b/vendor/cmd/go/internal/modload/search.go
index d67073e..9ce65f0 100644
--- a/vendor/cmd/go/internal/modload/search.go
+++ b/vendor/cmd/go/internal/modload/search.go
@@ -44,7 +44,7 @@
root = ModRoot
} else {
var err error
- root, err = fetch(mod)
+ root, _, err = fetch(mod)
if err != nil {
base.Errorf("go: %v", err)
continue
diff --git a/vendor/cmd/go/internal/mvs/mvs.go b/vendor/cmd/go/internal/mvs/mvs.go
index 03303d6..8ec9162 100644
--- a/vendor/cmd/go/internal/mvs/mvs.go
+++ b/vendor/cmd/go/internal/mvs/mvs.go
@@ -11,6 +11,7 @@
"sort"
"sync"
+ "cmd/go/internal/base"
"cmd/go/internal/module"
"cmd/go/internal/par"
)
@@ -99,11 +100,20 @@
mu.Unlock()
for _, r := range required {
+ if r.Path == "" {
+ base.Errorf("Required(%v) returned zero module in list", m)
+ continue
+ }
work.Add(r)
}
if upgrade != nil {
- work.Add(upgrade(m))
+ u := upgrade(m)
+ if u.Path == "" {
+ base.Errorf("Upgrade(%v) returned zero module", m)
+ return
+ }
+ work.Add(u)
}
})
diff --git a/vendor/cmd/go/internal/par/work.go b/vendor/cmd/go/internal/par/work.go
index a568c86..6543f1a 100644
--- a/vendor/cmd/go/internal/par/work.go
+++ b/vendor/cmd/go/internal/par/work.go
@@ -55,6 +55,7 @@
if n < 1 {
panic("par.Work.Do: n < 1")
}
+ n = 1
if w.running >= 1 {
panic("par.Work.Do: already called Do")
}
diff --git a/vendor/cmd/go/mod_test.go b/vendor/cmd/go/mod_test.go
index d94556b..dbff0ca 100644
--- a/vendor/cmd/go/mod_test.go
+++ b/vendor/cmd/go/mod_test.go
@@ -9,6 +9,7 @@
"internal/testenv"
"io/ioutil"
"os"
+ "os/exec"
"path/filepath"
"regexp"
"runtime"
@@ -590,8 +591,9 @@
module x
`), 0666))
tg.run("list")
- tg.grepStderr(`adding rsc.io/quote v1.5.2`, "should have added quote v1.5.2")
- tg.grepStderrNot(`v1.5.3-pre1`, "should not mention v1.5.3-pre1")
+ tg.run("list", "-m", "all")
+ tg.grepStdout(`quote v1.5.2$`, "should have added quote v1.5.2")
+ tg.grepStdoutNot(`v1.5.3-pre1`, "should not mention v1.5.3-pre1")
tg.run("list", "-m", "-versions", "rsc.io/quote")
want := "rsc.io/quote v1.0.0 v1.1.0 v1.2.0 v1.2.1 v1.3.0 v1.4.0 v1.5.0 v1.5.1 v1.5.2 v1.5.3-pre1\n"
@@ -659,13 +661,13 @@
tg.cd(filepath.Join(wd, "testdata/badmod"))
tg.runFail("get", "appengine")
- tg.grepStderr(`unrecognized import path \"appengine\"`, "expected appengine error ")
+ tg.grepStderr(`cannot find module providing package appengine`, "expected module error ")
tg.runFail("get", "x/y.z")
- tg.grepStderr(`unrecognized import path \"x/y.z\" \(import path does not begin with hostname\)`, "expected domain error")
+ tg.grepStderr(`cannot find module providing package x/y.z`, "expected module error")
tg.runFail("build")
tg.grepStderrNot("unknown module appengine: not a domain name", "expected nothing about appengine")
- tg.grepStderr("tcp.*nonexistent.rsc.io", "expected error for nonexistent.rsc.io")
+ tg.grepStderr("cannot find module providing package nonexistent.rsc.io", "expected error for nonexistent.rsc.io")
}
func TestModSync(t *testing.T) {
@@ -1289,3 +1291,67 @@
t.Fatal("produces duplicate imports")
}
}
+
+func TestModMultiVersion(t *testing.T) {
+ // TODO: Add tg.useGoModules, tg.mustHaveGoGet.
+ testenv.MustHaveExternalNetwork(t)
+ if _, err := exec.LookPath("git"); err != nil {
+ t.Skip("skipping because git binary not found")
+ }
+ tg := testgo(t)
+ tg.setenv("GO111MODULE", "on")
+ defer tg.cleanup()
+ tg.makeTempdir()
+ tg.setenv("GOPATH", tg.path("gp1"))
+
+ tg.must(os.MkdirAll(tg.path("quote"), 0777))
+ tg.cd(tg.path("quote"))
+
+ git := func(args ...string) {
+ t.Helper()
+ cmd := exec.Command("git", args...)
+ cmd.Dir = tg.path("quote")
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("git %v: %v\n%s", strings.Join(args, " "), err, out)
+ }
+ }
+
+ checkModules := func(dirs ...string) {
+ t.Helper()
+ tg.run("list", "-deps", "-f", "{{.ImportPath}}: {{.Dir}}")
+ for _, line := range strings.Split(tg.getStdout(), "\n") {
+ line = strings.Replace(line, `\`, `/`, -1) // windows!
+ if strings.HasPrefix(line, "rsc.io/quote: ") {
+ if strings.Contains(line, "/src/mod/") {
+ t.Fatalf("rsc.io/quote should not be from module cache: %v", line)
+ }
+ } else if strings.Contains(line, "rsc.io/quote") {
+ if !strings.Contains(line, "/src/mod/") {
+ t.Fatalf("rsc.io/quote/* should be from module cache: %v", line)
+ }
+ }
+ }
+ git("reset", "--hard") // reset go.mod to make git happy
+ git("clean", "-f") // delete go.sum to make git happy
+ }
+
+ git("clone", "https://github.com/rsc/quote", ".")
+ git("checkout", "v1.5.2")
+ checkModules()
+
+ git("checkout", "0d003b9") // wraps v2
+ checkModules() // looks up v2 from internet
+
+ git("checkout", "b44a0b1") // adds go.mod
+ checkModules() // knows which v2 to use (still needs download from internet, cached from last step)
+
+ git("checkout", "fe488b8") // adds broken v3 subdirectory
+ tg.run("list", "./...") // should ignore v3 because v3/go.mod exists
+
+ git("checkout", "a91498b") // wraps v3
+ checkModules() // looks up v3 from internet, not v3 subdirectory
+
+ git("checkout", "5d9f230") // adds go.mod
+ checkModules() // knows which v3 to use (still needs download from internet, cached from last step)
+}
