internal/imports/mod_cache.go - tools - Git at Google

 // Copyright 2019 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package imports

 import (
 	"context"
 	"fmt"
 	"path"
 	"path/filepath"
 	"strings"
 	"sync"

 	"golang.org/x/mod/module"
 	"golang.org/x/tools/internal/gopathwalk"
 	"golang.org/x/tools/internal/stdlib"
 )

 // To find packages to import, the resolver needs to know about all of
 // the packages that could be imported. This includes packages that are
 // already in modules that are in (1) the current module, (2) replace targets,
 // and (3) packages in the module cache. Packages in (1) and (2) may change over
 // time, as the client may edit the current module and locally replaced modules.
 // The module cache (which includes all of the packages in (3)) can only
 // ever be added to.
 //
 // The resolver can thus save state about packages in the module cache
 // and guarantee that this will not change over time. To obtain information
 // about new modules added to the module cache, the module cache should be
 // rescanned.
 //
 // It is OK to serve information about modules that have been deleted,
 // as they do still exist.
 // TODO(suzmue): can we share information with the caller about
 // what module needs to be downloaded to import this package?

 type directoryPackageStatus int

 const (
 	_ directoryPackageStatus = iota
 	directoryScanned
 	nameLoaded
 	exportsLoaded
 )

 // directoryPackageInfo holds (possibly incomplete) information about packages
 // contained in a given directory.
 type directoryPackageInfo struct {
 	// status indicates the extent to which this struct has been filled in.
 	status directoryPackageStatus
 	// err is non-nil when there was an error trying to reach status.
 	err error

 	// Set when status >= directoryScanned.

 	// dir is the absolute directory of this package.
 	dir      string
 	rootType gopathwalk.RootType
 	// nonCanonicalImportPath is the package's expected import path. It may
 	// not actually be importable at that path.
 	nonCanonicalImportPath string

 	// Module-related information.
 	moduleDir  string // The directory that is the module root of this dir.
 	moduleName string // The module name that contains this dir.

 	// Set when status >= nameLoaded.

 	packageName string // the package name, as declared in the source.

 	// Set when status >= exportsLoaded.
 	// TODO(rfindley): it's hard to see this, but exports depend implicitly on
 	// the default build context GOOS and GOARCH.
 	//
 	// We can make this explicit, and key exports by GOOS, GOARCH.
 	exports []stdlib.Symbol
 }

 // reachedStatus returns true when info has a status at least target and any error associated with
 // an attempt to reach target.
 func (info *directoryPackageInfo) reachedStatus(target directoryPackageStatus) (bool, error) {
 	if info.err == nil {
 		return info.status >= target, nil
 	}
 	if info.status == target {
 		return true, info.err
 	}
 	return true, nil
 }

 // DirInfoCache is a concurrency-safe map for storing information about
 // directories that may contain packages.
 //
 // The information in this cache is built incrementally. Entries are initialized in scan.
 // No new keys should be added in any other functions, as all directories containing
 // packages are identified in scan.
 //
 // Other functions, including loadExports and findPackage, may update entries in this cache
 // as they discover new things about the directory.
 //
 // The information in the cache is not expected to change for the cache's
 // lifetime, so there is no protection against competing writes. Users should
 // take care not to hold the cache across changes to the underlying files.
 type DirInfoCache struct {
 	mu sync.Mutex
 	// dirs stores information about packages in directories, keyed by absolute path.
 	dirs      map[string]*directoryPackageInfo
 	listeners map[*int]cacheListener
 }

 func NewDirInfoCache() *DirInfoCache {
 	return &DirInfoCache{
 		dirs:      make(map[string]*directoryPackageInfo),
 		listeners: make(map[*int]cacheListener),
 	}
 }

 type cacheListener func(directoryPackageInfo)

 // ScanAndListen calls listener on all the items in the cache, and on anything
 // newly added. The returned stop function waits for all in-flight callbacks to
 // finish and blocks new ones.
 func (d *DirInfoCache) ScanAndListen(ctx context.Context, listener cacheListener) func() {
 	ctx, cancel := context.WithCancel(ctx)

 	// Flushing out all the callbacks is tricky without knowing how many there
 	// are going to be. Setting an arbitrary limit makes it much easier.
 	const maxInFlight = 10
 	sema := make(chan struct{}, maxInFlight)
 	for i := 0; i < maxInFlight; i++ {
 		sema <- struct{}{}
 	}

 	cookie := new(int) // A unique ID we can use for the listener.

 	// We can't hold mu while calling the listener.
 	d.mu.Lock()
 	var keys []string
 	for key := range d.dirs {
 		keys = append(keys, key)
 	}
 	d.listeners[cookie] = func(info directoryPackageInfo) {
 		select {
 		case <-ctx.Done():
 			return
 		case <-sema:
 		}
 		listener(info)
 		sema <- struct{}{}
 	}
 	d.mu.Unlock()

 	stop := func() {
 		cancel()
 		d.mu.Lock()
 		delete(d.listeners, cookie)
 		d.mu.Unlock()
 		for i := 0; i < maxInFlight; i++ {
 			<-sema
 		}
 	}

 	// Process the pre-existing keys.
 	for _, k := range keys {
 		select {
 		case <-ctx.Done():
 			return stop
 		default:
 		}
 		if v, ok := d.Load(k); ok {
 			listener(v)
 		}
 	}

 	return stop
 }

 // Store stores the package info for dir.
 func (d *DirInfoCache) Store(dir string, info directoryPackageInfo) {
 	d.mu.Lock()
 	// TODO(rfindley, golang/go#59216): should we overwrite an existing entry?
 	// That seems incorrect as the cache should be idempotent.
 	_, old := d.dirs[dir]
 	d.dirs[dir] = &info
 	var listeners []cacheListener
 	for _, l := range d.listeners {
 		listeners = append(listeners, l)
 	}
 	d.mu.Unlock()

 	if !old {
 		for _, l := range listeners {
 			l(info)
 		}
 	}
 }

 // Load returns a copy of the directoryPackageInfo for absolute directory dir.
 func (d *DirInfoCache) Load(dir string) (directoryPackageInfo, bool) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	info, ok := d.dirs[dir]
 	if !ok {
 		return directoryPackageInfo{}, false
 	}
 	return *info, true
 }

 // Keys returns the keys currently present in d.
 func (d *DirInfoCache) Keys() (keys []string) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	for key := range d.dirs {
 		keys = append(keys, key)
 	}
 	return keys
 }

 func (d *DirInfoCache) CachePackageName(info directoryPackageInfo) (string, error) {
 	if loaded, err := info.reachedStatus(nameLoaded); loaded {
 		return info.packageName, err
 	}
 	if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
 		return "", fmt.Errorf("cannot read package name, scan error: %v", err)
 	}
 	info.packageName, info.err = packageDirToName(info.dir)
 	info.status = nameLoaded
 	d.Store(info.dir, info)
 	return info.packageName, info.err
 }

 func (d *DirInfoCache) CacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []stdlib.Symbol, error) {
 	if reached, _ := info.reachedStatus(exportsLoaded); reached {
 		return info.packageName, info.exports, info.err
 	}
 	if reached, err := info.reachedStatus(nameLoaded); reached && err != nil {
 		return "", nil, err
 	}
 	info.packageName, info.exports, info.err = loadExportsFromFiles(ctx, env, info.dir, false)
 	if info.err == context.Canceled || info.err == context.DeadlineExceeded {
 		return info.packageName, info.exports, info.err
 	}
 	// The cache structure wants things to proceed linearly. We can skip a
 	// step here, but only if we succeed.
 	if info.status == nameLoaded || info.err == nil {
 		info.status = exportsLoaded
 	} else {
 		info.status = nameLoaded
 	}
 	d.Store(info.dir, info)
 	return info.packageName, info.exports, info.err
 }

 // ScanModuleCache walks the given directory, which must be a GOMODCACHE value,
 // for directory package information, storing the results in cache.
 func ScanModuleCache(dir string, cache *DirInfoCache, logf func(string, ...any)) {
 	// Note(rfindley): it's hard to see, but this function attempts to implement
 	// just the side effects on cache of calling PrimeCache with a ProcessEnv
 	// that has the given dir as its GOMODCACHE.
 	//
 	// Teasing out the control flow, we see that we can avoid any handling of
 	// vendor/ and can infer module info entirely from the path, simplifying the
 	// logic here.

 	root := gopathwalk.Root{
 		Path: filepath.Clean(dir),
 		Type: gopathwalk.RootModuleCache,
 	}

 	directoryInfo := func(root gopathwalk.Root, dir string) directoryPackageInfo {
 		// This is a copy of ModuleResolver.scanDirForPackage, trimmed down to
 		// logic that applies to a module cache directory.

 		subdir := ""
 		if dir != root.Path {
 			subdir = dir[len(root.Path)+len("/"):]
 		}

 		matches := modCacheRegexp.FindStringSubmatch(subdir)
 		if len(matches) == 0 {
 			return directoryPackageInfo{
 				status: directoryScanned,
 				err:    fmt.Errorf("invalid module cache path: %v", subdir),
 			}
 		}
 		modPath, err := module.UnescapePath(filepath.ToSlash(matches[1]))
 		if err != nil {
 			if logf != nil {
 				logf("decoding module cache path %q: %v", subdir, err)
 			}
 			return directoryPackageInfo{
 				status: directoryScanned,
 				err:    fmt.Errorf("decoding module cache path %q: %v", subdir, err),
 			}
 		}
 		importPath := path.Join(modPath, filepath.ToSlash(matches[3]))
 		index := strings.Index(dir, matches[1]+"@"+matches[2])
 		modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
 		modName := readModName(filepath.Join(modDir, "go.mod"))
 		return directoryPackageInfo{
 			status:                 directoryScanned,
 			dir:                    dir,
 			rootType:               root.Type,
 			nonCanonicalImportPath: importPath,
 			moduleDir:              modDir,
 			moduleName:             modName,
 		}
 	}

 	add := func(root gopathwalk.Root, dir string) {
 		info := directoryInfo(root, dir)
 		cache.Store(info.dir, info)
 	}

 	skip := func(_ gopathwalk.Root, dir string) bool {
 		// Skip directories that have already been scanned.
 		//
 		// Note that gopathwalk only adds "package" directories, which must contain
 		// a .go file, and all such package directories in the module cache are
 		// immutable. So if we can load a dir, it can be skipped.
 		info, ok := cache.Load(dir)
 		if !ok {
 			return false
 		}
 		packageScanned, _ := info.reachedStatus(directoryScanned)
 		return packageScanned
 	}

 	gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: logf, ModulesEnabled: true})
 }
	// Copyright 2019 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package imports

	import (
	"context"
	"fmt"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"golang.org/x/mod/module"
	"golang.org/x/tools/internal/gopathwalk"
	"golang.org/x/tools/internal/stdlib"
	)

	// To find packages to import, the resolver needs to know about all of
	// the packages that could be imported. This includes packages that are
	// already in modules that are in (1) the current module, (2) replace targets,
	// and (3) packages in the module cache. Packages in (1) and (2) may change over
	// time, as the client may edit the current module and locally replaced modules.
	// The module cache (which includes all of the packages in (3)) can only
	// ever be added to.
	//
	// The resolver can thus save state about packages in the module cache
	// and guarantee that this will not change over time. To obtain information
	// about new modules added to the module cache, the module cache should be
	// rescanned.
	//
	// It is OK to serve information about modules that have been deleted,
	// as they do still exist.
	// TODO(suzmue): can we share information with the caller about
	// what module needs to be downloaded to import this package?

	type directoryPackageStatus int

	const (
	_ directoryPackageStatus = iota
	directoryScanned
	nameLoaded
	exportsLoaded
	)

	// directoryPackageInfo holds (possibly incomplete) information about packages
	// contained in a given directory.
	type directoryPackageInfo struct {
	// status indicates the extent to which this struct has been filled in.
	status directoryPackageStatus
	// err is non-nil when there was an error trying to reach status.
	err error

	// Set when status >= directoryScanned.

	// dir is the absolute directory of this package.
	dir string
	rootType gopathwalk.RootType
	// nonCanonicalImportPath is the package's expected import path. It may
	// not actually be importable at that path.
	nonCanonicalImportPath string

	// Module-related information.
	moduleDir string // The directory that is the module root of this dir.
	moduleName string // The module name that contains this dir.

	// Set when status >= nameLoaded.

	packageName string // the package name, as declared in the source.

	// Set when status >= exportsLoaded.
	// TODO(rfindley): it's hard to see this, but exports depend implicitly on
	// the default build context GOOS and GOARCH.
	//
	// We can make this explicit, and key exports by GOOS, GOARCH.
	exports []stdlib.Symbol
	}

	// reachedStatus returns true when info has a status at least target and any error associated with
	// an attempt to reach target.
	func (info *directoryPackageInfo) reachedStatus(target directoryPackageStatus) (bool, error) {
	if info.err == nil {
	return info.status >= target, nil
	}
	if info.status == target {
	return true, info.err
	}
	return true, nil
	}

	// DirInfoCache is a concurrency-safe map for storing information about
	// directories that may contain packages.
	//
	// The information in this cache is built incrementally. Entries are initialized in scan.
	// No new keys should be added in any other functions, as all directories containing
	// packages are identified in scan.
	//
	// Other functions, including loadExports and findPackage, may update entries in this cache
	// as they discover new things about the directory.
	//
	// The information in the cache is not expected to change for the cache's
	// lifetime, so there is no protection against competing writes. Users should
	// take care not to hold the cache across changes to the underlying files.
	type DirInfoCache struct {
	mu sync.Mutex
	// dirs stores information about packages in directories, keyed by absolute path.
	dirs map[string]*directoryPackageInfo
	listeners map[*int]cacheListener
	}

	func NewDirInfoCache() *DirInfoCache {
	return &DirInfoCache{
	dirs: make(map[string]*directoryPackageInfo),
	listeners: make(map[*int]cacheListener),
	}
	}

	type cacheListener func(directoryPackageInfo)

	// ScanAndListen calls listener on all the items in the cache, and on anything
	// newly added. The returned stop function waits for all in-flight callbacks to
	// finish and blocks new ones.
	func (d *DirInfoCache) ScanAndListen(ctx context.Context, listener cacheListener) func() {
	ctx, cancel := context.WithCancel(ctx)

	// Flushing out all the callbacks is tricky without knowing how many there
	// are going to be. Setting an arbitrary limit makes it much easier.
	const maxInFlight = 10
	sema := make(chan struct{}, maxInFlight)
	for i := 0; i < maxInFlight; i++ {
	sema <- struct{}{}
	}

	cookie := new(int) // A unique ID we can use for the listener.

	// We can't hold mu while calling the listener.
	d.mu.Lock()
	var keys []string
	for key := range d.dirs {
	keys = append(keys, key)
	}
	d.listeners[cookie] = func(info directoryPackageInfo) {
	select {
	case <-ctx.Done():
	return
	case <-sema:
	}
	listener(info)
	sema <- struct{}{}
	}
	d.mu.Unlock()

	stop := func() {
	cancel()
	d.mu.Lock()
	delete(d.listeners, cookie)
	d.mu.Unlock()
	for i := 0; i < maxInFlight; i++ {
	<-sema
	}
	}

	// Process the pre-existing keys.
	for _, k := range keys {
	select {
	case <-ctx.Done():
	return stop
	default:
	}
	if v, ok := d.Load(k); ok {
	listener(v)
	}
	}

	return stop
	}

	// Store stores the package info for dir.
	func (d *DirInfoCache) Store(dir string, info directoryPackageInfo) {
	d.mu.Lock()
	// TODO(rfindley, golang/go#59216): should we overwrite an existing entry?
	// That seems incorrect as the cache should be idempotent.
	_, old := d.dirs[dir]
	d.dirs[dir] = &info
	var listeners []cacheListener
	for _, l := range d.listeners {
	listeners = append(listeners, l)
	}
	d.mu.Unlock()

	if !old {
	for _, l := range listeners {
	l(info)
	}
	}
	}

	// Load returns a copy of the directoryPackageInfo for absolute directory dir.
	func (d *DirInfoCache) Load(dir string) (directoryPackageInfo, bool) {
	d.mu.Lock()
	defer d.mu.Unlock()
	info, ok := d.dirs[dir]
	if !ok {
	return directoryPackageInfo{}, false
	}
	return *info, true
	}

	// Keys returns the keys currently present in d.
	func (d *DirInfoCache) Keys() (keys []string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	for key := range d.dirs {
	keys = append(keys, key)
	}
	return keys
	}

	func (d *DirInfoCache) CachePackageName(info directoryPackageInfo) (string, error) {
	if loaded, err := info.reachedStatus(nameLoaded); loaded {
	return info.packageName, err
	}
	if scanned, err := info.reachedStatus(directoryScanned); !scanned \|\| err != nil {
	return "", fmt.Errorf("cannot read package name, scan error: %v", err)
	}
	info.packageName, info.err = packageDirToName(info.dir)
	info.status = nameLoaded
	d.Store(info.dir, info)
	return info.packageName, info.err
	}

	func (d DirInfoCache) CacheExports(ctx context.Context, env ProcessEnv, info directoryPackageInfo) (string, []stdlib.Symbol, error) {
	if reached, _ := info.reachedStatus(exportsLoaded); reached {
	return info.packageName, info.exports, info.err
	}
	if reached, err := info.reachedStatus(nameLoaded); reached && err != nil {
	return "", nil, err
	}
	info.packageName, info.exports, info.err = loadExportsFromFiles(ctx, env, info.dir, false)
	if info.err == context.Canceled \|\| info.err == context.DeadlineExceeded {
	return info.packageName, info.exports, info.err
	}
	// The cache structure wants things to proceed linearly. We can skip a
	// step here, but only if we succeed.
	if info.status == nameLoaded \|\| info.err == nil {
	info.status = exportsLoaded
	} else {
	info.status = nameLoaded
	}
	d.Store(info.dir, info)
	return info.packageName, info.exports, info.err
	}

	// ScanModuleCache walks the given directory, which must be a GOMODCACHE value,
	// for directory package information, storing the results in cache.
	func ScanModuleCache(dir string, cache *DirInfoCache, logf func(string, ...any)) {
	// Note(rfindley): it's hard to see, but this function attempts to implement
	// just the side effects on cache of calling PrimeCache with a ProcessEnv
	// that has the given dir as its GOMODCACHE.
	//
	// Teasing out the control flow, we see that we can avoid any handling of
	// vendor/ and can infer module info entirely from the path, simplifying the
	// logic here.

	root := gopathwalk.Root{
	Path: filepath.Clean(dir),
	Type: gopathwalk.RootModuleCache,
	}

	directoryInfo := func(root gopathwalk.Root, dir string) directoryPackageInfo {
	// This is a copy of ModuleResolver.scanDirForPackage, trimmed down to
	// logic that applies to a module cache directory.

	subdir := ""
	if dir != root.Path {
	subdir = dir[len(root.Path)+len("/"):]
	}

	matches := modCacheRegexp.FindStringSubmatch(subdir)
	if len(matches) == 0 {
	return directoryPackageInfo{
	status: directoryScanned,
	err: fmt.Errorf("invalid module cache path: %v", subdir),
	}
	}
	modPath, err := module.UnescapePath(filepath.ToSlash(matches[1]))
	if err != nil {
	if logf != nil {
	logf("decoding module cache path %q: %v", subdir, err)
	}
	return directoryPackageInfo{
	status: directoryScanned,
	err: fmt.Errorf("decoding module cache path %q: %v", subdir, err),
	}
	}
	importPath := path.Join(modPath, filepath.ToSlash(matches[3]))
	index := strings.Index(dir, matches[1]+"@"+matches[2])
	modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
	modName := readModName(filepath.Join(modDir, "go.mod"))
	return directoryPackageInfo{
	status: directoryScanned,
	dir: dir,
	rootType: root.Type,
	nonCanonicalImportPath: importPath,
	moduleDir: modDir,
	moduleName: modName,
	}
	}

	add := func(root gopathwalk.Root, dir string) {
	info := directoryInfo(root, dir)
	cache.Store(info.dir, info)
	}

	skip := func(_ gopathwalk.Root, dir string) bool {
	// Skip directories that have already been scanned.
	//
	// Note that gopathwalk only adds "package" directories, which must contain
	// a .go file, and all such package directories in the module cache are
	// immutable. So if we can load a dir, it can be skipped.
	info, ok := cache.Load(dir)
	if !ok {
	return false
	}
	packageScanned, _ := info.reachedStatus(directoryScanned)
	return packageScanned
	}

	gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: logf, ModulesEnabled: true})
	}