| // Copyright 2018 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package modload |
| |
| // This file contains the module-mode package loader, as well as some accessory |
| // functions pertaining to the package import graph. |
| // |
| // There are two exported entry points into package loading — LoadPackages and |
| // ImportFromFiles — both implemented in terms of loadFromRoots, which itself |
| // manipulates an instance of the loader struct. |
| // |
| // Although most of the loading state is maintained in the loader struct, |
| // one key piece - the build list - is a global, so that it can be modified |
| // separate from the loading operation, such as during "go get" |
| // upgrades/downgrades or in "go mod" operations. |
| // TODO(#40775): It might be nice to make the loader take and return |
| // a buildList rather than hard-coding use of the global. |
| // |
| // Loading is an iterative process. On each iteration, we try to load the |
| // requested packages and their transitive imports, then try to resolve modules |
| // for any imported packages that are still missing. |
| // |
| // The first step of each iteration identifies a set of “root” packages. |
| // Normally the root packages are exactly those matching the named pattern |
| // arguments. However, for the "all" meta-pattern, the final set of packages is |
| // computed from the package import graph, and therefore cannot be an initial |
| // input to loading that graph. Instead, the root packages for the "all" pattern |
| // are those contained in the main module, and allPatternIsRoot parameter to the |
| // loader instructs it to dynamically expand those roots to the full "all" |
| // pattern as loading progresses. |
| // |
| // The pkgInAll flag on each loadPkg instance tracks whether that |
| // package is known to match the "all" meta-pattern. |
| // A package matches the "all" pattern if: |
| // - it is in the main module, or |
| // - it is imported by any test in the main module, or |
| // - it is imported by another package in "all", or |
| // - the main module specifies a go version ≤ 1.15, and the package is imported |
| // by a *test of* another package in "all". |
| // |
| // When we implement lazy loading, we will record the modules providing packages |
| // in "all" even when we are only loading individual packages, so we set the |
| // pkgInAll flag regardless of the whether the "all" pattern is a root. |
| // (This is necessary to maintain the “import invariant” described in |
| // https://golang.org/design/36460-lazy-module-loading.) |
| // |
| // Because "go mod vendor" prunes out the tests of vendored packages, the |
| // behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same |
| // as the "all" pattern (regardless of the -mod flag) in 1.16+. |
| // The allClosesOverTests parameter to the loader indicates whether the "all" |
| // pattern should close over tests (as in Go 1.11–1.15) or stop at only those |
| // packages transitively imported by the packages and tests in the main module |
| // ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). |
| // |
| // Note that it is possible for a loaded package NOT to be in "all" even when we |
| // are loading the "all" pattern. For example, packages that are transitive |
| // dependencies of other roots named on the command line must be loaded, but are |
| // not in "all". (The mod_notall test illustrates this behavior.) |
| // Similarly, if the LoadTests flag is set but the "all" pattern does not close |
| // over test dependencies, then when we load the test of a package that is in |
| // "all" but outside the main module, the dependencies of that test will not |
| // necessarily themselves be in "all". That configuration does not arise in Go |
| // 1.11–1.15, but it will be possible with lazy loading in Go 1.16+. |
| // |
| // Loading proceeds from the roots, using a parallel work-queue with a limit on |
| // the amount of active work (to avoid saturating disks, CPU cores, and/or |
| // network connections). Each package is added to the queue the first time it is |
| // imported by another package. When we have finished identifying the imports of |
| // a package, we add the test for that package if it is needed. A test may be |
| // needed if: |
| // - the package matches a root pattern and tests of the roots were requested, or |
| // - the package is in the main module and the "all" pattern is requested |
| // (because the "all" pattern includes the dependencies of tests in the main |
| // module), or |
| // - the package is in "all" and the definition of "all" we are using includes |
| // dependencies of tests (as is the case in Go ≤1.15). |
| // |
| // After all available packages have been loaded, we examine the results to |
| // identify any requested or imported packages that are still missing, and if |
| // so, which modules we could add to the module graph in order to make the |
| // missing packages available. We add those to the module graph and iterate, |
| // until either all packages resolve successfully or we cannot identify any |
| // module that would resolve any remaining missing package. |
| // |
| // If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) |
| // and all requested packages are in "all", then loading completes in a single |
| // iteration. |
| // TODO(bcmills): We should also be able to load in a single iteration if the |
| // requested packages all come from modules that are themselves tidy, regardless |
| // of whether those packages are in "all". Today, that requires two iterations |
| // if those packages are not found in existing dependencies of the main module. |
| |
| import ( |
| "bytes" |
| "context" |
| "errors" |
| "fmt" |
| "go/build" |
| "io/fs" |
| "os" |
| "path" |
| pathpkg "path" |
| "path/filepath" |
| "reflect" |
| "runtime" |
| "sort" |
| "strings" |
| "sync" |
| "sync/atomic" |
| |
| "cmd/go/internal/base" |
| "cmd/go/internal/cfg" |
| "cmd/go/internal/fsys" |
| "cmd/go/internal/imports" |
| "cmd/go/internal/modfetch" |
| "cmd/go/internal/mvs" |
| "cmd/go/internal/par" |
| "cmd/go/internal/search" |
| "cmd/go/internal/str" |
| |
| "golang.org/x/mod/module" |
| ) |
| |
| // loaded is the most recently-used package loader. |
| // It holds details about individual packages. |
| var loaded *loader |
| |
| // PackageOpts control the behavior of the LoadPackages function. |
| type PackageOpts struct { |
| // Tags are the build tags in effect (as interpreted by the |
| // cmd/go/internal/imports package). |
| // If nil, treated as equivalent to imports.Tags(). |
| Tags map[string]bool |
| |
| // ResolveMissingImports indicates that we should attempt to add module |
| // dependencies as needed to resolve imports of packages that are not found. |
| // |
| // For commands that support the -mod flag, resolving imports may still fail |
| // if the flag is set to "readonly" (the default) or "vendor". |
| ResolveMissingImports bool |
| |
| // AllowPackage, if non-nil, is called after identifying the module providing |
| // each package. If AllowPackage returns a non-nil error, that error is set |
| // for the package, and the imports and test of that package will not be |
| // loaded. |
| // |
| // AllowPackage may be invoked concurrently by multiple goroutines, |
| // and may be invoked multiple times for a given package path. |
| AllowPackage func(ctx context.Context, path string, mod module.Version) error |
| |
| // LoadTests loads the test dependencies of each package matching a requested |
| // pattern. If ResolveMissingImports is also true, test dependencies will be |
| // resolved if missing. |
| LoadTests bool |
| |
| // UseVendorAll causes the "all" package pattern to be interpreted as if |
| // running "go mod vendor" (or building with "-mod=vendor"). |
| // |
| // Once lazy loading is implemented, this will be a no-op for modules that |
| // declare 'go 1.16' or higher. |
| UseVendorAll bool |
| |
| // AllowErrors indicates that LoadPackages should not terminate the process if |
| // an error occurs. |
| AllowErrors bool |
| |
| // SilenceErrors indicates that LoadPackages should not print errors |
| // that occur while loading packages. SilenceErrors implies AllowErrors. |
| SilenceErrors bool |
| |
| // SilenceUnmatchedWarnings suppresses the warnings normally emitted for |
| // patterns that did not match any packages. |
| SilenceUnmatchedWarnings bool |
| } |
| |
| // LoadPackages identifies the set of packages matching the given patterns and |
| // loads the packages in the import graph rooted at that set. |
| func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { |
| LoadModFile(ctx) |
| if opts.Tags == nil { |
| opts.Tags = imports.Tags() |
| } |
| |
| patterns = search.CleanPatterns(patterns) |
| matches = make([]*search.Match, 0, len(patterns)) |
| allPatternIsRoot := false |
| for _, pattern := range patterns { |
| matches = append(matches, search.NewMatch(pattern)) |
| if pattern == "all" { |
| allPatternIsRoot = true |
| } |
| } |
| |
| updateMatches := func(ld *loader) { |
| for _, m := range matches { |
| switch { |
| case m.IsLocal(): |
| // Evaluate list of file system directories on first iteration. |
| if m.Dirs == nil { |
| matchLocalDirs(m) |
| } |
| |
| // Make a copy of the directory list and translate to import paths. |
| // Note that whether a directory corresponds to an import path |
| // changes as the build list is updated, and a directory can change |
| // from not being in the build list to being in it and back as |
| // the exact version of a particular module increases during |
| // the loader iterations. |
| m.Pkgs = m.Pkgs[:0] |
| for _, dir := range m.Dirs { |
| pkg, err := resolveLocalPackage(dir) |
| if err != nil { |
| if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) { |
| continue // Don't include "builtin" or GOROOT/src in wildcard patterns. |
| } |
| |
| // If we're outside of a module, ensure that the failure mode |
| // indicates that. |
| ModRoot() |
| |
| if ld != nil { |
| m.AddError(err) |
| } |
| continue |
| } |
| m.Pkgs = append(m.Pkgs, pkg) |
| } |
| |
| case m.IsLiteral(): |
| m.Pkgs = []string{m.Pattern()} |
| |
| case strings.Contains(m.Pattern(), "..."): |
| m.Errs = m.Errs[:0] |
| matchPackages(ctx, m, opts.Tags, includeStd, buildList) |
| |
| case m.Pattern() == "all": |
| if ld == nil { |
| // The initial roots are the packages in the main module. |
| // loadFromRoots will expand that to "all". |
| m.Errs = m.Errs[:0] |
| matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) |
| } else { |
| // Starting with the packages in the main module, |
| // enumerate the full list of "all". |
| m.Pkgs = ld.computePatternAll() |
| } |
| |
| case m.Pattern() == "std" || m.Pattern() == "cmd": |
| if m.Pkgs == nil { |
| m.MatchPackages() // Locate the packages within GOROOT/src. |
| } |
| |
| default: |
| panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern())) |
| } |
| } |
| } |
| |
| loaded = loadFromRoots(loaderParams{ |
| PackageOpts: opts, |
| |
| allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, |
| allPatternIsRoot: allPatternIsRoot, |
| |
| listRoots: func() (roots []string) { |
| updateMatches(nil) |
| for _, m := range matches { |
| roots = append(roots, m.Pkgs...) |
| } |
| return roots |
| }, |
| }) |
| |
| // One last pass to finalize wildcards. |
| updateMatches(loaded) |
| |
| // Report errors, if any. |
| checkMultiplePaths() |
| for _, pkg := range loaded.pkgs { |
| if pkg.err != nil { |
| if pkg.flags.has(pkgInAll) { |
| if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) { |
| imErr.inAll = true |
| } else if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { |
| sumErr.inAll = true |
| } |
| } |
| |
| if !opts.SilenceErrors { |
| if opts.AllowErrors { |
| fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) |
| } else { |
| base.Errorf("%s: %v", pkg.stackText(), pkg.err) |
| } |
| } |
| } |
| if !pkg.isTest() { |
| loadedPackages = append(loadedPackages, pkg.path) |
| } |
| } |
| if !opts.SilenceErrors { |
| // Also list errors in matching patterns (such as directory permission |
| // errors for wildcard patterns). |
| for _, match := range matches { |
| for _, err := range match.Errs { |
| if opts.AllowErrors { |
| fmt.Fprintf(os.Stderr, "%v\n", err) |
| } else { |
| base.Errorf("%v", err) |
| } |
| } |
| } |
| } |
| base.ExitIfErrors() |
| |
| if !opts.SilenceUnmatchedWarnings { |
| search.WarnUnmatched(matches) |
| } |
| |
| // Success! Update go.mod (if needed) and return the results. |
| WriteGoMod() |
| sort.Strings(loadedPackages) |
| return matches, loadedPackages |
| } |
| |
| // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories |
| // outside of the standard library and active modules. |
| func matchLocalDirs(m *search.Match) { |
| if !m.IsLocal() { |
| panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) |
| } |
| |
| if i := strings.Index(m.Pattern(), "..."); i >= 0 { |
| // The pattern is local, but it is a wildcard. Its packages will |
| // only resolve to paths if they are inside of the standard |
| // library, the main module, or some dependency of the main |
| // module. Verify that before we walk the filesystem: a filesystem |
| // walk in a directory like /var or /etc can be very expensive! |
| dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3])) |
| absDir := dir |
| if !filepath.IsAbs(dir) { |
| absDir = filepath.Join(base.Cwd, dir) |
| } |
| if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(absDir) == "" { |
| m.Dirs = []string{} |
| m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) |
| return |
| } |
| } |
| |
| m.MatchDirs() |
| } |
| |
| // resolveLocalPackage resolves a filesystem path to a package path. |
| func resolveLocalPackage(dir string) (string, error) { |
| var absDir string |
| if filepath.IsAbs(dir) { |
| absDir = filepath.Clean(dir) |
| } else { |
| absDir = filepath.Join(base.Cwd, dir) |
| } |
| |
| bp, err := cfg.BuildContext.ImportDir(absDir, 0) |
| if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) { |
| // golang.org/issue/32917: We should resolve a relative path to a |
| // package path only if the relative path actually contains the code |
| // for that package. |
| // |
| // If the named directory does not exist or contains no Go files, |
| // the package does not exist. |
| // Other errors may affect package loading, but not resolution. |
| if _, err := fsys.Stat(absDir); err != nil { |
| if os.IsNotExist(err) { |
| // Canonicalize OS-specific errors to errDirectoryNotFound so that error |
| // messages will be easier for users to search for. |
| return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} |
| } |
| return "", err |
| } |
| if _, noGo := err.(*build.NoGoError); noGo { |
| // A directory that does not contain any Go source files — even ignored |
| // ones! — is not a Go package, and we can't resolve it to a package |
| // path because that path could plausibly be provided by some other |
| // module. |
| // |
| // Any other error indicates that the package “exists” (at least in the |
| // sense that it cannot exist in any other module), but has some other |
| // problem (such as a syntax error). |
| return "", err |
| } |
| } |
| |
| if modRoot != "" && absDir == modRoot { |
| if absDir == cfg.GOROOTsrc { |
| return "", errPkgIsGorootSrc |
| } |
| return targetPrefix, nil |
| } |
| |
| // Note: The checks for @ here are just to avoid misinterpreting |
| // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). |
| // It's not strictly necessary but helpful to keep the checks. |
| if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { |
| suffix := filepath.ToSlash(absDir[len(modRoot):]) |
| if strings.HasPrefix(suffix, "/vendor/") { |
| if cfg.BuildMod != "vendor" { |
| return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) |
| } |
| |
| readVendorList() |
| pkg := strings.TrimPrefix(suffix, "/vendor/") |
| if _, ok := vendorPkgModule[pkg]; !ok { |
| return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) |
| } |
| return pkg, nil |
| } |
| |
| if targetPrefix == "" { |
| pkg := strings.TrimPrefix(suffix, "/") |
| if pkg == "builtin" { |
| // "builtin" is a pseudo-package with a real source file. |
| // It's not included in "std", so it shouldn't resolve from "." |
| // within module "std" either. |
| return "", errPkgIsBuiltin |
| } |
| return pkg, nil |
| } |
| |
| pkg := targetPrefix + suffix |
| if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil { |
| return "", err |
| } else if !ok { |
| return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg} |
| } |
| return pkg, nil |
| } |
| |
| if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { |
| pkg := filepath.ToSlash(sub) |
| if pkg == "builtin" { |
| return "", errPkgIsBuiltin |
| } |
| return pkg, nil |
| } |
| |
| pkg := pathInModuleCache(absDir) |
| if pkg == "" { |
| return "", fmt.Errorf("directory %s outside available modules", base.ShortPath(absDir)) |
| } |
| return pkg, nil |
| } |
| |
| var ( |
| errDirectoryNotFound = errors.New("directory not found") |
| errPkgIsGorootSrc = errors.New("GOROOT/src is not an importable package") |
| errPkgIsBuiltin = errors.New(`"builtin" is a pseudo-package, not an importable package`) |
| ) |
| |
| // pathInModuleCache returns the import path of the directory dir, |
| // if dir is in the module cache copy of a module in our build list. |
| func pathInModuleCache(dir string) string { |
| tryMod := func(m module.Version) (string, bool) { |
| var root string |
| var err error |
| if repl := Replacement(m); repl.Path != "" && repl.Version == "" { |
| root = repl.Path |
| if !filepath.IsAbs(root) { |
| root = filepath.Join(ModRoot(), root) |
| } |
| } else if repl.Path != "" { |
| root, err = modfetch.DownloadDir(repl) |
| } else { |
| root, err = modfetch.DownloadDir(m) |
| } |
| if err != nil { |
| return "", false |
| } |
| |
| sub := search.InDir(dir, root) |
| if sub == "" { |
| return "", false |
| } |
| sub = filepath.ToSlash(sub) |
| if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { |
| return "", false |
| } |
| |
| return path.Join(m.Path, filepath.ToSlash(sub)), true |
| } |
| |
| for _, m := range buildList[1:] { |
| if importPath, ok := tryMod(m); ok { |
| // checkMultiplePaths ensures that a module can be used for at most one |
| // requirement, so this must be it. |
| return importPath |
| } |
| } |
| return "" |
| } |
| |
| // ImportFromFiles adds modules to the build list as needed |
| // to satisfy the imports in the named Go source files. |
| func ImportFromFiles(ctx context.Context, gofiles []string) { |
| LoadModFile(ctx) |
| |
| tags := imports.Tags() |
| imports, testImports, err := imports.ScanFiles(gofiles, tags) |
| if err != nil { |
| base.Fatalf("go: %v", err) |
| } |
| |
| loaded = loadFromRoots(loaderParams{ |
| PackageOpts: PackageOpts{ |
| Tags: tags, |
| ResolveMissingImports: true, |
| }, |
| allClosesOverTests: index.allPatternClosesOverTests(), |
| listRoots: func() (roots []string) { |
| roots = append(roots, imports...) |
| roots = append(roots, testImports...) |
| return roots |
| }, |
| }) |
| WriteGoMod() |
| } |
| |
| // DirImportPath returns the effective import path for dir, |
| // provided it is within the main module, or else returns ".". |
| func DirImportPath(dir string) string { |
| if !HasModRoot() { |
| return "." |
| } |
| LoadModFile(context.TODO()) |
| |
| if !filepath.IsAbs(dir) { |
| dir = filepath.Join(base.Cwd, dir) |
| } else { |
| dir = filepath.Clean(dir) |
| } |
| |
| if dir == modRoot { |
| return targetPrefix |
| } |
| if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { |
| suffix := filepath.ToSlash(dir[len(modRoot):]) |
| if strings.HasPrefix(suffix, "/vendor/") { |
| return strings.TrimPrefix(suffix, "/vendor/") |
| } |
| return targetPrefix + suffix |
| } |
| return "." |
| } |
| |
| // TargetPackages returns the list of packages in the target (top-level) module |
| // matching pattern, which may be relative to the working directory, under all |
| // build tag settings. |
| func TargetPackages(ctx context.Context, pattern string) *search.Match { |
| // TargetPackages is relative to the main module, so ensure that the main |
| // module is a thing that can contain packages. |
| LoadModFile(ctx) |
| ModRoot() |
| |
| m := search.NewMatch(pattern) |
| matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) |
| return m |
| } |
| |
| // ImportMap returns the actual package import path |
| // for an import path found in source code. |
| // If the given import path does not appear in the source code |
| // for the packages that have been loaded, ImportMap returns the empty string. |
| func ImportMap(path string) string { |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| return "" |
| } |
| return pkg.path |
| } |
| |
| // PackageDir returns the directory containing the source code |
| // for the package named by the import path. |
| func PackageDir(path string) string { |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| return "" |
| } |
| return pkg.dir |
| } |
| |
| // PackageModule returns the module providing the package named by the import path. |
| func PackageModule(path string) module.Version { |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| return module.Version{} |
| } |
| return pkg.mod |
| } |
| |
| // PackageImports returns the imports for the package named by the import path. |
| // Test imports will be returned as well if tests were loaded for the package |
| // (i.e., if "all" was loaded or if LoadTests was set and the path was matched |
| // by a command line argument). PackageImports will return nil for |
| // unknown package paths. |
| func PackageImports(path string) (imports, testImports []string) { |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| return nil, nil |
| } |
| imports = make([]string, len(pkg.imports)) |
| for i, p := range pkg.imports { |
| imports[i] = p.path |
| } |
| if pkg.test != nil { |
| testImports = make([]string, len(pkg.test.imports)) |
| for i, p := range pkg.test.imports { |
| testImports[i] = p.path |
| } |
| } |
| return imports, testImports |
| } |
| |
| // Lookup returns the source directory, import path, and any loading error for |
| // the package at path as imported from the package in parentDir. |
| // Lookup requires that one of the Load functions in this package has already |
| // been called. |
| func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) { |
| if path == "" { |
| panic("Lookup called with empty package path") |
| } |
| |
| if parentIsStd { |
| path = loaded.stdVendor(parentPath, path) |
| } |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| // The loader should have found all the relevant paths. |
| // There are a few exceptions, though: |
| // - during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports |
| // end up here to canonicalize the import paths. |
| // - during any load, non-loaded packages like "unsafe" end up here. |
| // - during any load, build-injected dependencies like "runtime/cgo" end up here. |
| // - because we ignore appengine/* in the module loader, |
| // the dependencies of any actual appengine/* library end up here. |
| dir := findStandardImportPath(path) |
| if dir != "" { |
| return dir, path, nil |
| } |
| return "", "", errMissing |
| } |
| return pkg.dir, pkg.path, pkg.err |
| } |
| |
| // A loader manages the process of loading information about |
| // the required packages for a particular build, |
| // checking that the packages are available in the module set, |
| // and updating the module set if needed. |
| type loader struct { |
| loaderParams |
| |
| work *par.Queue |
| |
| // reset on each iteration |
| roots []*loadPkg |
| pkgCache *par.Cache // package path (string) → *loadPkg |
| pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks |
| |
| // computed at end of iterations |
| direct map[string]bool // imported directly by main module |
| } |
| |
| // loaderParams configure the packages loaded by, and the properties reported |
| // by, a loader instance. |
| type loaderParams struct { |
| PackageOpts |
| |
| allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? |
| allPatternIsRoot bool // Is the "all" pattern an additional root? |
| |
| listRoots func() []string |
| } |
| |
| func (ld *loader) reset() { |
| select { |
| case <-ld.work.Idle(): |
| default: |
| panic("loader.reset when not idle") |
| } |
| |
| ld.roots = nil |
| ld.pkgCache = new(par.Cache) |
| ld.pkgs = nil |
| } |
| |
| // A loadPkg records information about a single loaded package. |
| type loadPkg struct { |
| // Populated at construction time: |
| path string // import path |
| testOf *loadPkg |
| |
| // Populated at construction time and updated by (*loader).applyPkgFlags: |
| flags atomicLoadPkgFlags |
| |
| // Populated by (*loader).load: |
| mod module.Version // module providing package |
| dir string // directory containing source code |
| err error // error loading package |
| imports []*loadPkg // packages imported by this one |
| testImports []string // test-only imports, saved for use by pkg.test. |
| inStd bool |
| |
| // Populated by (*loader).pkgTest: |
| testOnce sync.Once |
| test *loadPkg |
| |
| // Populated by postprocessing in (*loader).buildStacks: |
| stack *loadPkg // package importing this one in minimal import stack for this pkg |
| } |
| |
| // loadPkgFlags is a set of flags tracking metadata about a package. |
| type loadPkgFlags int8 |
| |
| const ( |
| // pkgInAll indicates that the package is in the "all" package pattern, |
| // regardless of whether we are loading the "all" package pattern. |
| // |
| // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller |
| // who set the last of those flags must propagate the pkgInAll marking to all |
| // of the imports of the marked package. |
| // |
| // A test is marked with pkgInAll if that test would promote the packages it |
| // imports to be in "all" (such as when the test is itself within the main |
| // module, or when ld.allClosesOverTests is true). |
| pkgInAll loadPkgFlags = 1 << iota |
| |
| // pkgIsRoot indicates that the package matches one of the root package |
| // patterns requested by the caller. |
| // |
| // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, |
| // the caller who set the last of those flags must populate a test for the |
| // package (in the pkg.test field). |
| // |
| // If the "all" pattern is included as a root, then non-test packages in "all" |
| // are also roots (and must be marked pkgIsRoot). |
| pkgIsRoot |
| |
| // pkgImportsLoaded indicates that the imports and testImports fields of a |
| // loadPkg have been populated. |
| pkgImportsLoaded |
| ) |
| |
| // has reports whether all of the flags in cond are set in f. |
| func (f loadPkgFlags) has(cond loadPkgFlags) bool { |
| return f&cond == cond |
| } |
| |
| // An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be |
| // added atomically. |
| type atomicLoadPkgFlags struct { |
| bits int32 |
| } |
| |
| // update sets the given flags in af (in addition to any flags already set). |
| // |
| // update returns the previous flag state so that the caller may determine which |
| // flags were newly-set. |
| func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { |
| for { |
| old := atomic.LoadInt32(&af.bits) |
| new := old | int32(flags) |
| if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { |
| return loadPkgFlags(old) |
| } |
| } |
| } |
| |
| // has reports whether all of the flags in cond are set in af. |
| func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { |
| return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond |
| } |
| |
| // isTest reports whether pkg is a test of another package. |
| func (pkg *loadPkg) isTest() bool { |
| return pkg.testOf != nil |
| } |
| |
| var errMissing = errors.New("cannot find package") |
| |
| // loadFromRoots attempts to load the build graph needed to process a set of |
| // root packages and their dependencies. |
| // |
| // The set of root packages is returned by the params.listRoots function, and |
| // expanded to the full set of packages by tracing imports (and possibly tests) |
| // as needed. |
| func loadFromRoots(params loaderParams) *loader { |
| ld := &loader{ |
| loaderParams: params, |
| work: par.NewQueue(runtime.GOMAXPROCS(0)), |
| } |
| |
| var err error |
| reqs := &mvsReqs{buildList: buildList} |
| buildList, err = mvs.BuildList(Target, reqs) |
| if err != nil { |
| base.Fatalf("go: %v", err) |
| } |
| |
| addedModuleFor := make(map[string]bool) |
| for { |
| ld.reset() |
| |
| // Load the root packages and their imports. |
| // Note: the returned roots can change on each iteration, |
| // since the expansion of package patterns depends on the |
| // build list we're using. |
| inRoots := map[*loadPkg]bool{} |
| for _, path := range ld.listRoots() { |
| root := ld.pkg(path, pkgIsRoot) |
| if !inRoots[root] { |
| ld.roots = append(ld.roots, root) |
| inRoots[root] = true |
| } |
| } |
| |
| // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, |
| // which adds tests (and test dependencies) as needed. |
| // |
| // When all of the work in the queue has completed, we'll know that the |
| // transitive closure of dependencies has been loaded. |
| <-ld.work.Idle() |
| |
| ld.buildStacks() |
| |
| if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { |
| // We've loaded as much as we can without resolving missing imports. |
| break |
| } |
| modAddedBy := ld.resolveMissingImports(addedModuleFor) |
| if len(modAddedBy) == 0 { |
| break |
| } |
| |
| // Recompute buildList with all our additions. |
| reqs = &mvsReqs{buildList: buildList} |
| buildList, err = mvs.BuildList(Target, reqs) |
| if err != nil { |
| // If an error was found in a newly added module, report the package |
| // import stack instead of the module requirement stack. Packages |
| // are more descriptive. |
| if err, ok := err.(*mvs.BuildListError); ok { |
| if pkg := modAddedBy[err.Module()]; pkg != nil { |
| base.Fatalf("go: %s: %v", pkg.stackText(), err.Err) |
| } |
| } |
| base.Fatalf("go: %v", err) |
| } |
| } |
| base.ExitIfErrors() |
| |
| // Compute directly referenced dependency modules. |
| ld.direct = make(map[string]bool) |
| for _, pkg := range ld.pkgs { |
| if pkg.mod == Target { |
| for _, dep := range pkg.imports { |
| if dep.mod.Path != "" { |
| ld.direct[dep.mod.Path] = true |
| } |
| } |
| } |
| } |
| |
| // If we didn't scan all of the imports from the main module, or didn't use |
| // imports.AnyTags, then we didn't necessarily load every package that |
| // contributes “direct” imports — so we can't safely mark existing |
| // dependencies as indirect-only. |
| // Conservatively mark those dependencies as direct. |
| if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { |
| for _, r := range modFile.Require { |
| if !r.Indirect { |
| ld.direct[r.Mod.Path] = true |
| } |
| } |
| } |
| |
| return ld |
| } |
| |
| // resolveMissingImports adds module dependencies to the global build list |
| // in order to resolve missing packages from pkgs. |
| // |
| // The newly-resolved packages are added to the addedModuleFor map, and |
| // resolveMissingImports returns a map from each newly-added module version to |
| // the first package for which that module was added. |
| func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { |
| var needPkgs []*loadPkg |
| for _, pkg := range ld.pkgs { |
| if pkg.err == nil { |
| continue |
| } |
| if pkg.isTest() { |
| // If we are missing a test, we are also missing its non-test version, and |
| // we should only add the missing import once. |
| continue |
| } |
| if !errors.As(pkg.err, new(*ImportMissingError)) { |
| // Leave other errors for Import or load.Packages to report. |
| continue |
| } |
| |
| needPkgs = append(needPkgs, pkg) |
| |
| pkg := pkg |
| ld.work.Add(func() { |
| pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) |
| }) |
| } |
| <-ld.work.Idle() |
| |
| modAddedBy = map[module.Version]*loadPkg{} |
| for _, pkg := range needPkgs { |
| if pkg.err != nil { |
| continue |
| } |
| |
| fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) |
| if addedModuleFor[pkg.path] { |
| // TODO(bcmills): This should only be an error if pkg.mod is the same |
| // version we already tried to add previously. |
| base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) |
| } |
| if modAddedBy[pkg.mod] == nil { |
| modAddedBy[pkg.mod] = pkg |
| buildList = append(buildList, pkg.mod) |
| } |
| } |
| |
| return modAddedBy |
| } |
| |
| // pkg locates the *loadPkg for path, creating and queuing it for loading if |
| // needed, and updates its state to reflect the given flags. |
| // |
| // The imports of the returned *loadPkg will be loaded asynchronously in the |
| // ld.work queue, and its test (if requested) will also be populated once |
| // imports have been resolved. When ld.work goes idle, all transitive imports of |
| // the requested package (and its test, if requested) will have been loaded. |
| func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { |
| if flags.has(pkgImportsLoaded) { |
| panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") |
| } |
| |
| pkg := ld.pkgCache.Do(path, func() interface{} { |
| pkg := &loadPkg{ |
| path: path, |
| } |
| ld.applyPkgFlags(pkg, flags) |
| |
| ld.work.Add(func() { ld.load(pkg) }) |
| return pkg |
| }).(*loadPkg) |
| |
| ld.applyPkgFlags(pkg, flags) |
| return pkg |
| } |
| |
| // applyPkgFlags updates pkg.flags to set the given flags and propagate the |
| // (transitive) effects of those flags, possibly loading or enqueueing further |
| // packages as a result. |
| func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { |
| if flags == 0 { |
| return |
| } |
| |
| if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { |
| // This package matches a root pattern by virtue of being in "all". |
| flags |= pkgIsRoot |
| } |
| |
| old := pkg.flags.update(flags) |
| new := old | flags |
| if new == old || !new.has(pkgImportsLoaded) { |
| // We either didn't change the state of pkg, or we don't know anything about |
| // its dependencies yet. Either way, we can't usefully load its test or |
| // update its dependencies. |
| return |
| } |
| |
| if !pkg.isTest() { |
| // Check whether we should add (or update the flags for) a test for pkg. |
| // ld.pkgTest is idempotent and extra invocations are inexpensive, |
| // so it's ok if we call it more than is strictly necessary. |
| wantTest := false |
| switch { |
| case ld.allPatternIsRoot && pkg.mod == Target: |
| // We are loading the "all" pattern, which includes packages imported by |
| // tests in the main module. This package is in the main module, so we |
| // need to identify the imports of its test even if LoadTests is not set. |
| // |
| // (We will filter out the extra tests explicitly in computePatternAll.) |
| wantTest = true |
| |
| case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): |
| // This variant of the "all" pattern includes imports of tests of every |
| // package that is itself in "all", and pkg is in "all", so its test is |
| // also in "all" (as above). |
| wantTest = true |
| |
| case ld.LoadTests && new.has(pkgIsRoot): |
| // LoadTest explicitly requests tests of “the root packages”. |
| wantTest = true |
| } |
| |
| if wantTest { |
| var testFlags loadPkgFlags |
| if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { |
| // Tests of packages in the main module are in "all", in the sense that |
| // they cause the packages they import to also be in "all". So are tests |
| // of packages in "all" if "all" closes over test dependencies. |
| testFlags |= pkgInAll |
| } |
| ld.pkgTest(pkg, testFlags) |
| } |
| } |
| |
| if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { |
| // We have just marked pkg with pkgInAll, or we have just loaded its |
| // imports, or both. Now is the time to propagate pkgInAll to the imports. |
| for _, dep := range pkg.imports { |
| ld.applyPkgFlags(dep, pkgInAll) |
| } |
| } |
| } |
| |
| // load loads an individual package. |
| func (ld *loader) load(pkg *loadPkg) { |
| if strings.Contains(pkg.path, "@") { |
| // Leave for error during load. |
| return |
| } |
| if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { |
| // Leave for error during load. |
| // (Module mode does not allow local imports.) |
| return |
| } |
| |
| if search.IsMetaPackage(pkg.path) { |
| pkg.err = &invalidImportError{ |
| importPath: pkg.path, |
| err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), |
| } |
| return |
| } |
| |
| pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path, buildList) |
| if pkg.dir == "" { |
| return |
| } |
| if pkg.mod == Target { |
| // Go ahead and mark pkg as in "all". This provides the invariant that a |
| // package that is *only* imported by other packages in "all" is always |
| // marked as such before loading its imports. |
| // |
| // We don't actually rely on that invariant at the moment, but it may |
| // improve efficiency somewhat and makes the behavior a bit easier to reason |
| // about (by reducing churn on the flag bits of dependencies), and costs |
| // essentially nothing (these atomic flag ops are essentially free compared |
| // to scanning source code for imports). |
| ld.applyPkgFlags(pkg, pkgInAll) |
| } |
| if ld.AllowPackage != nil { |
| if err := ld.AllowPackage(context.TODO(), pkg.path, pkg.mod); err != nil { |
| pkg.err = err |
| } |
| } |
| |
| imports, testImports, err := scanDir(pkg.dir, ld.Tags) |
| if err != nil { |
| pkg.err = err |
| return |
| } |
| |
| pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") |
| |
| pkg.imports = make([]*loadPkg, 0, len(imports)) |
| var importFlags loadPkgFlags |
| if pkg.flags.has(pkgInAll) { |
| importFlags = pkgInAll |
| } |
| for _, path := range imports { |
| if pkg.inStd { |
| // Imports from packages in "std" and "cmd" should resolve using |
| // GOROOT/src/vendor even when "std" is not the main module. |
| path = ld.stdVendor(pkg.path, path) |
| } |
| pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) |
| } |
| pkg.testImports = testImports |
| |
| ld.applyPkgFlags(pkg, pkgImportsLoaded) |
| } |
| |
| // pkgTest locates the test of pkg, creating it if needed, and updates its state |
| // to reflect the given flags. |
| // |
| // pkgTest requires that the imports of pkg have already been loaded (flagged |
| // with pkgImportsLoaded). |
| func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { |
| if pkg.isTest() { |
| panic("pkgTest called on a test package") |
| } |
| |
| createdTest := false |
| pkg.testOnce.Do(func() { |
| pkg.test = &loadPkg{ |
| path: pkg.path, |
| testOf: pkg, |
| mod: pkg.mod, |
| dir: pkg.dir, |
| err: pkg.err, |
| inStd: pkg.inStd, |
| } |
| ld.applyPkgFlags(pkg.test, testFlags) |
| createdTest = true |
| }) |
| |
| test := pkg.test |
| if createdTest { |
| test.imports = make([]*loadPkg, 0, len(pkg.testImports)) |
| var importFlags loadPkgFlags |
| if test.flags.has(pkgInAll) { |
| importFlags = pkgInAll |
| } |
| for _, path := range pkg.testImports { |
| if pkg.inStd { |
| path = ld.stdVendor(test.path, path) |
| } |
| test.imports = append(test.imports, ld.pkg(path, importFlags)) |
| } |
| pkg.testImports = nil |
| ld.applyPkgFlags(test, pkgImportsLoaded) |
| } else { |
| ld.applyPkgFlags(test, testFlags) |
| } |
| |
| return test |
| } |
| |
| // stdVendor returns the canonical import path for the package with the given |
| // path when imported from the standard-library package at parentPath. |
| func (ld *loader) stdVendor(parentPath, path string) string { |
| if search.IsStandardImportPath(path) { |
| return path |
| } |
| |
| if str.HasPathPrefix(parentPath, "cmd") { |
| if Target.Path != "cmd" { |
| vendorPath := pathpkg.Join("cmd", "vendor", path) |
| if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { |
| return vendorPath |
| } |
| } |
| } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { |
| // If we are outside of the 'std' module, resolve imports from within 'std' |
| // to the vendor directory. |
| // |
| // Do the same for importers beginning with the prefix 'vendor/' even if we |
| // are *inside* of the 'std' module: the 'vendor/' packages that resolve |
| // globally from GOROOT/src/vendor (and are listed as part of 'go list std') |
| // are distinct from the real module dependencies, and cannot import |
| // internal packages from the real module. |
| // |
| // (Note that although the 'vendor/' packages match the 'std' *package* |
| // pattern, they are not part of the std *module*, and do not affect |
| // 'go mod tidy' and similar module commands when working within std.) |
| vendorPath := pathpkg.Join("vendor", path) |
| if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { |
| return vendorPath |
| } |
| } |
| |
| // Not vendored: resolve from modules. |
| return path |
| } |
| |
| // computePatternAll returns the list of packages matching pattern "all", |
| // starting with a list of the import paths for the packages in the main module. |
| func (ld *loader) computePatternAll() (all []string) { |
| for _, pkg := range ld.pkgs { |
| if pkg.flags.has(pkgInAll) && !pkg.isTest() { |
| all = append(all, pkg.path) |
| } |
| } |
| sort.Strings(all) |
| return all |
| } |
| |
| // scanDir is like imports.ScanDir but elides known magic imports from the list, |
| // so that we do not go looking for packages that don't really exist. |
| // |
| // The standard magic import is "C", for cgo. |
| // |
| // The only other known magic imports are appengine and appengine/*. |
| // These are so old that they predate "go get" and did not use URL-like paths. |
| // Most code today now uses google.golang.org/appengine instead, |
| // but not all code has been so updated. When we mostly ignore build tags |
| // during "go vendor", we look into "// +build appengine" files and |
| // may see these legacy imports. We drop them so that the module |
| // search does not look for modules to try to satisfy them. |
| func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) { |
| imports_, testImports, err = imports.ScanDir(dir, tags) |
| |
| filter := func(x []string) []string { |
| w := 0 |
| for _, pkg := range x { |
| if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") && |
| pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") { |
| x[w] = pkg |
| w++ |
| } |
| } |
| return x[:w] |
| } |
| |
| return filter(imports_), filter(testImports), err |
| } |
| |
| // buildStacks computes minimal import stacks for each package, |
| // for use in error messages. When it completes, packages that |
| // are part of the original root set have pkg.stack == nil, |
| // and other packages have pkg.stack pointing at the next |
| // package up the import stack in their minimal chain. |
| // As a side effect, buildStacks also constructs ld.pkgs, |
| // the list of all packages loaded. |
| func (ld *loader) buildStacks() { |
| if len(ld.pkgs) > 0 { |
| panic("buildStacks") |
| } |
| for _, pkg := range ld.roots { |
| pkg.stack = pkg // sentinel to avoid processing in next loop |
| ld.pkgs = append(ld.pkgs, pkg) |
| } |
| for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop |
| pkg := ld.pkgs[i] |
| for _, next := range pkg.imports { |
| if next.stack == nil { |
| next.stack = pkg |
| ld.pkgs = append(ld.pkgs, next) |
| } |
| } |
| if next := pkg.test; next != nil && next.stack == nil { |
| next.stack = pkg |
| ld.pkgs = append(ld.pkgs, next) |
| } |
| } |
| for _, pkg := range ld.roots { |
| pkg.stack = nil |
| } |
| } |
| |
| // stackText builds the import stack text to use when |
| // reporting an error in pkg. It has the general form |
| // |
| // root imports |
| // other imports |
| // other2 tested by |
| // other2.test imports |
| // pkg |
| // |
| func (pkg *loadPkg) stackText() string { |
| var stack []*loadPkg |
| for p := pkg; p != nil; p = p.stack { |
| stack = append(stack, p) |
| } |
| |
| var buf bytes.Buffer |
| for i := len(stack) - 1; i >= 0; i-- { |
| p := stack[i] |
| fmt.Fprint(&buf, p.path) |
| if p.testOf != nil { |
| fmt.Fprint(&buf, ".test") |
| } |
| if i > 0 { |
| if stack[i-1].testOf == p { |
| fmt.Fprint(&buf, " tested by\n\t") |
| } else { |
| fmt.Fprint(&buf, " imports\n\t") |
| } |
| } |
| } |
| return buf.String() |
| } |
| |
| // why returns the text to use in "go mod why" output about the given package. |
| // It is less ornate than the stackText but contains the same information. |
| func (pkg *loadPkg) why() string { |
| var buf strings.Builder |
| var stack []*loadPkg |
| for p := pkg; p != nil; p = p.stack { |
| stack = append(stack, p) |
| } |
| |
| for i := len(stack) - 1; i >= 0; i-- { |
| p := stack[i] |
| if p.testOf != nil { |
| fmt.Fprintf(&buf, "%s.test\n", p.testOf.path) |
| } else { |
| fmt.Fprintf(&buf, "%s\n", p.path) |
| } |
| } |
| return buf.String() |
| } |
| |
| // Why returns the "go mod why" output stanza for the given package, |
| // without the leading # comment. |
| // The package graph must have been loaded already, usually by LoadPackages. |
| // If there is no reason for the package to be in the current build, |
| // Why returns an empty string. |
| func Why(path string) string { |
| pkg, ok := loaded.pkgCache.Get(path).(*loadPkg) |
| if !ok { |
| return "" |
| } |
| return pkg.why() |
| } |
| |
| // WhyDepth returns the number of steps in the Why listing. |
| // If there is no reason for the package to be in the current build, |
| // WhyDepth returns 0. |
| func WhyDepth(path string) int { |
| n := 0 |
| pkg, _ := loaded.pkgCache.Get(path).(*loadPkg) |
| for p := pkg; p != nil; p = p.stack { |
| n++ |
| } |
| return n |
| } |