internal/lsp: support directory inclusion/exclusion filters
Users working in large repositories may want to include only selected
directories in their workspace to avoid memory usage and performance
slowdowns. Add support for inclusion/exclusion filters that control what
directories are searched for workspace packages and modules. Packages
that are excluded by the filter may still be loaded as non-workspace
packages if other things depend on them.
For a description of the option's syntax, see the documentation.
Note that because we don't have any way to communicate the filters to
packages.Load, we still run go list on the unfiltered workspace scope,
then throw away the irrelevant packages. That may cost us, especially in
workspaces with many files.
Comments on the naming welcome. Also, if you know any places I may have
missed applying the filter, please do tell. One thing I thought of is
file watching, but that's covered because allKnownSubdirs works off of
workspace files and those are already filtered.
Possible enhancements:
- Support glob patterns.
- Apply filters during the goimports scan.
- Figure out how to apply the filters to packages.Load. I don't know
how to do it while still being build system neutral though.
Closes golang/go#42473, assuming none of the enhancements are required.
Change-Id: I9006a7a361dc3bb3c11f78b05ff84981813035a0
Reviewed-on: https://go-review.googlesource.com/c/tools/+/275253
Trust: Heschi Kreinick <heschi@google.com>
Run-TryBot: Heschi Kreinick <heschi@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Rebecca Stambler <rstambler@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
diff --git a/internal/lsp/cache/load.go b/internal/lsp/cache/load.go
index 530c156..62426b1 100644
--- a/internal/lsp/cache/load.go
+++ b/internal/lsp/cache/load.go
@@ -159,6 +159,11 @@
if isTestMain(pkg, s.view.gocache) {
continue
}
+ // Skip filtered packages. They may be added anyway if they're
+ // dependencies of non-filtered packages.
+ if s.view.allFilesExcluded(pkg) {
+ continue
+ }
// Set the metadata for this package.
m, err := s.setMetadata(ctx, packagePath(pkg.PkgPath), pkg, cfg, map[packageID]struct{}{})
if err != nil {
diff --git a/internal/lsp/cache/session.go b/internal/lsp/cache/session.go
index f3bf1fb..9e3c01f 100644
--- a/internal/lsp/cache/session.go
+++ b/internal/lsp/cache/session.go
@@ -175,14 +175,14 @@
}
root := folder
if options.ExpandWorkspaceToModule {
- root, err = findWorkspaceRoot(ctx, root, s, options.ExperimentalWorkspaceModule)
+ root, err = findWorkspaceRoot(ctx, root, s, pathExcludedByFilterFunc(options), options.ExperimentalWorkspaceModule)
if err != nil {
return nil, nil, func() {}, err
}
}
// Build the gopls workspace, collecting active modules in the view.
- workspace, err := newWorkspace(ctx, root, s, ws.userGo111Module == off, options.ExperimentalWorkspaceModule)
+ workspace, err := newWorkspace(ctx, root, s, pathExcludedByFilterFunc(options), ws.userGo111Module == off, options.ExperimentalWorkspaceModule)
if err != nil {
return nil, nil, func() {}, err
}
diff --git a/internal/lsp/cache/snapshot.go b/internal/lsp/cache/snapshot.go
index 421686c..c023def 100644
--- a/internal/lsp/cache/snapshot.go
+++ b/internal/lsp/cache/snapshot.go
@@ -1608,12 +1608,12 @@
// BuildGoplsMod generates a go.mod file for all modules in the workspace. It
// bypasses any existing gopls.mod.
-func BuildGoplsMod(ctx context.Context, root span.URI, fs source.FileSource) (*modfile.File, error) {
- allModules, err := findModules(ctx, root, 0)
+func BuildGoplsMod(ctx context.Context, root span.URI, s source.Snapshot) (*modfile.File, error) {
+ allModules, err := findModules(ctx, root, pathExcludedByFilterFunc(s.View().Options()), 0)
if err != nil {
return nil, err
}
- return buildWorkspaceModFile(ctx, allModules, fs)
+ return buildWorkspaceModFile(ctx, allModules, s)
}
// TODO(rfindley): move this to workspacemodule.go
diff --git a/internal/lsp/cache/view.go b/internal/lsp/cache/view.go
index 81f5f6a..f6bcae5 100644
--- a/internal/lsp/cache/view.go
+++ b/internal/lsp/cache/view.go
@@ -23,6 +23,7 @@
"golang.org/x/mod/modfile"
"golang.org/x/mod/semver"
+ "golang.org/x/tools/go/packages"
"golang.org/x/tools/internal/event"
"golang.org/x/tools/internal/gocommand"
"golang.org/x/tools/internal/imports"
@@ -242,6 +243,9 @@
if !reflect.DeepEqual(a.Env, b.Env) {
return false
}
+ if !reflect.DeepEqual(a.DirectoryFilters, b.DirectoryFilters) {
+ return false
+ }
aBuildFlags := make([]string, len(a.BuildFlags))
bBuildFlags := make([]string, len(b.BuildFlags))
copy(aBuildFlags, a.BuildFlags)
@@ -323,7 +327,16 @@
}
func (v *View) contains(uri span.URI) bool {
- return source.InDir(v.rootURI.Filename(), uri.Filename()) || source.InDir(v.folder.Filename(), uri.Filename())
+ inRoot := source.InDir(v.rootURI.Filename(), uri.Filename())
+ inFolder := source.InDir(v.folder.Filename(), uri.Filename())
+ if !inRoot && !inFolder {
+ return false
+ }
+ // Filters are applied relative to the workspace folder.
+ if inFolder {
+ return !pathExcludedByFilter(strings.TrimPrefix(uri.Filename(), v.folder.Filename()), v.Options())
+ }
+ return true
}
func (v *View) mapFile(uri span.URI, f *fileBase) {
@@ -699,7 +712,7 @@
// Otherwise, it returns folder.
// TODO (rFindley): move this to workspace.go
// TODO (rFindley): simplify this once workspace modules are enabled by default.
-func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSource, experimental bool) (span.URI, error) {
+func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSource, excludePath func(string) bool, experimental bool) (span.URI, error) {
patterns := []string{"go.mod"}
if experimental {
patterns = []string{"gopls.mod", "go.mod"}
@@ -720,7 +733,7 @@
}
// ...else we should check if there's exactly one nested module.
- all, err := findModules(ctx, folder, 2)
+ all, err := findModules(ctx, folder, excludePath, 2)
if err == errExhausted {
// Fall-back behavior: if we don't find any modules after searching 10000
// files, assume there are none.
@@ -915,3 +928,43 @@
vendorEnabled := modFile.Go != nil && modFile.Go.Version != "" && semver.Compare("v"+modFile.Go.Version, "v1.14") >= 0
return vendorEnabled, nil
}
+
+func (v *View) allFilesExcluded(pkg *packages.Package) bool {
+ opts := v.Options()
+ folder := filepath.ToSlash(v.folder.Filename())
+ for _, f := range pkg.GoFiles {
+ f = filepath.ToSlash(f)
+ if !strings.HasPrefix(f, folder) {
+ return false
+ }
+ if !pathExcludedByFilter(strings.TrimPrefix(f, folder), opts) {
+ return false
+ }
+ }
+ return true
+}
+
+func pathExcludedByFilterFunc(opts *source.Options) func(string) bool {
+ return func(path string) bool {
+ return pathExcludedByFilter(path, opts)
+ }
+}
+
+func pathExcludedByFilter(path string, opts *source.Options) bool {
+ path = strings.TrimPrefix(filepath.ToSlash(path), "/")
+
+ excluded := false
+ for _, filter := range opts.DirectoryFilters {
+ op, prefix := filter[0], filter[1:]
+ // Non-empty prefixes have to be precise directory matches.
+ if prefix != "" {
+ prefix = prefix + "/"
+ path = path + "/"
+ }
+ if !strings.HasPrefix(path, prefix) {
+ continue
+ }
+ excluded = op == '-'
+ }
+ return excluded
+}
diff --git a/internal/lsp/cache/view_test.go b/internal/lsp/cache/view_test.go
index 1f66c81..054d410 100644
--- a/internal/lsp/cache/view_test.go
+++ b/internal/lsp/cache/view_test.go
@@ -95,7 +95,8 @@
ctx := context.Background()
rel := fake.RelativeTo(dir)
folderURI := span.URIFromPath(rel.AbsPath(test.folder))
- got, err := findWorkspaceRoot(ctx, folderURI, osFileSource{}, test.experimental)
+ excludeNothing := func(string) bool { return false }
+ got, err := findWorkspaceRoot(ctx, folderURI, osFileSource{}, excludeNothing, test.experimental)
if err != nil {
t.Fatal(err)
}
@@ -210,3 +211,49 @@
}
}
}
+
+func TestFilters(t *testing.T) {
+ tests := []struct {
+ filters []string
+ included []string
+ excluded []string
+ }{
+ {
+ included: []string{"x"},
+ },
+ {
+ filters: []string{"-"},
+ excluded: []string{"x", "x/a"},
+ },
+ {
+ filters: []string{"-x", "+y"},
+ included: []string{"y", "y/a", "z"},
+ excluded: []string{"x", "x/a"},
+ },
+ {
+ filters: []string{"-x", "+x/y", "-x/y/z"},
+ included: []string{"x/y", "x/y/a", "a"},
+ excluded: []string{"x", "x/a", "x/y/z/a"},
+ },
+ {
+ filters: []string{"+foobar", "-foo"},
+ included: []string{"foobar", "foobar/a"},
+ excluded: []string{"foo", "foo/a"},
+ },
+ }
+
+ for _, tt := range tests {
+ opts := &source.Options{}
+ opts.DirectoryFilters = tt.filters
+ for _, inc := range tt.included {
+ if pathExcludedByFilter(inc, opts) {
+ t.Errorf("filters %q excluded %v, wanted included", tt.filters, inc)
+ }
+ }
+ for _, exc := range tt.excluded {
+ if !pathExcludedByFilter(exc, opts) {
+ t.Errorf("filters %q included %v, wanted excluded", tt.filters, exc)
+ }
+ }
+ }
+}
diff --git a/internal/lsp/cache/workspace.go b/internal/lsp/cache/workspace.go
index 3a3477b..217d31f 100644
--- a/internal/lsp/cache/workspace.go
+++ b/internal/lsp/cache/workspace.go
@@ -54,6 +54,7 @@
// across multiple snapshots.
type workspace struct {
root span.URI
+ excludePath func(string) bool
moduleSource workspaceSource
// activeModFiles holds the active go.mod files.
@@ -81,7 +82,7 @@
wsDirs map[span.URI]struct{}
}
-func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, go111moduleOff bool, experimental bool) (*workspace, error) {
+func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, excludePath func(string) bool, go111moduleOff bool, experimental bool) (*workspace, error) {
// In experimental mode, the user may have a gopls.mod file that defines
// their workspace.
if experimental {
@@ -97,6 +98,7 @@
}
return &workspace{
root: root,
+ excludePath: excludePath,
activeModFiles: activeModFiles,
knownModFiles: activeModFiles,
mod: file,
@@ -106,7 +108,7 @@
}
// Otherwise, in all other modes, search for all of the go.mod files in the
// workspace.
- knownModFiles, err := findModules(ctx, root, 0)
+ knownModFiles, err := findModules(ctx, root, excludePath, 0)
if err != nil {
return nil, err
}
@@ -114,6 +116,7 @@
if go111moduleOff {
return &workspace{
root: root,
+ excludePath: excludePath,
moduleSource: legacyWorkspace,
knownModFiles: knownModFiles,
go111moduleOff: true,
@@ -127,6 +130,7 @@
}
return &workspace{
root: root,
+ excludePath: excludePath,
activeModFiles: activeModFiles,
knownModFiles: knownModFiles,
moduleSource: legacyWorkspace,
@@ -134,6 +138,7 @@
}
return &workspace{
root: root,
+ excludePath: excludePath,
activeModFiles: knownModFiles,
knownModFiles: knownModFiles,
moduleSource: fileSystemWorkspace,
@@ -279,7 +284,7 @@
} else {
// gopls.mod is deleted. search for modules again.
moduleSource = fileSystemWorkspace
- knownModFiles, err = findModules(ctx, w.root, 0)
+ knownModFiles, err = findModules(ctx, w.root, w.excludePath, 0)
// the modFile is no longer valid.
if err != nil {
event.Error(ctx, "finding file system modules", err)
@@ -335,6 +340,7 @@
// Any change to modules triggers a new version.
return &workspace{
root: w.root,
+ excludePath: w.excludePath,
moduleSource: moduleSource,
activeModFiles: activeModFiles,
knownModFiles: knownModFiles,
@@ -438,7 +444,7 @@
// searching stops once modLimit modules have been found.
//
// TODO(rfindley): consider overlays.
-func findModules(ctx context.Context, root span.URI, modLimit int) (map[span.URI]struct{}, error) {
+func findModules(ctx context.Context, root span.URI, excludePath func(string) bool, modLimit int) (map[span.URI]struct{}, error) {
// Walk the view's folder to find all modules in the view.
modFiles := make(map[span.URI]struct{})
searched := 0
@@ -455,7 +461,8 @@
suffix := strings.TrimPrefix(path, root.Filename())
switch {
case checkIgnored(suffix),
- strings.Contains(filepath.ToSlash(suffix), "/vendor/"):
+ strings.Contains(filepath.ToSlash(suffix), "/vendor/"),
+ excludePath(suffix):
return filepath.SkipDir
}
}
diff --git a/internal/lsp/cache/workspace_test.go b/internal/lsp/cache/workspace_test.go
index dfd543d..c0d36a7 100644
--- a/internal/lsp/cache/workspace_test.go
+++ b/internal/lsp/cache/workspace_test.go
@@ -185,7 +185,8 @@
root := span.URIFromPath(dir)
fs := osFileSource{}
- w, err := newWorkspace(ctx, root, fs, false, !test.legacyMode)
+ excludeNothing := func(string) bool { return false }
+ w, err := newWorkspace(ctx, root, fs, excludeNothing, false, !test.legacyMode)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/lsp/fake/editor.go b/internal/lsp/fake/editor.go
index afe47a6..e42c507 100644
--- a/internal/lsp/fake/editor.go
+++ b/internal/lsp/fake/editor.go
@@ -96,6 +96,8 @@
// the PID. This can only be set by one test.
SendPID bool
+ DirectoryFilters []string
+
VerboseOutput bool
}
@@ -197,7 +199,9 @@
if e.Config.BuildFlags != nil {
config["buildFlags"] = e.Config.BuildFlags
}
-
+ if e.Config.DirectoryFilters != nil {
+ config["directoryFilters"] = e.Config.DirectoryFilters
+ }
if e.Config.CodeLenses != nil {
config["codelenses"] = e.Config.CodeLenses
}
diff --git a/internal/lsp/source/api_json.go b/internal/lsp/source/api_json.go
index 550adf3..820268c 100755
--- a/internal/lsp/source/api_json.go
+++ b/internal/lsp/source/api_json.go
@@ -257,6 +257,13 @@
},
Default: "\"Dynamic\"",
},
+ {
+ Name: "directoryFilters",
+ Type: "[]string",
+ Doc: "directoryFilters can be used to exclude unwanted directories from the\nworkspace. By default, all directories are included. Filters are an\noperator, `+` to include and `-` to exclude, followed by a path prefix\nrelative to the workspace folder. They are evaluated in order, and\nthe last filter that applies to a path controls whether it is included.\nThe path prefix can be empty, so an initial `-` excludes everything.\n\nExamples:\nExclude node_modules: `-node_modules`\nInclude only project_a: `-` (exclude everything), `+project_a`\nInclude only project_a, but not node_modules inside it: `-`, `+project_a`, `-project_a/node_modules`\n",
+ EnumValues: nil,
+ Default: "[]",
+ },
},
},
Commands: []*CommandJSON{
diff --git a/internal/lsp/source/options.go b/internal/lsp/source/options.go
index b396dbb..797c205 100644
--- a/internal/lsp/source/options.go
+++ b/internal/lsp/source/options.go
@@ -7,6 +7,7 @@
import (
"context"
"fmt"
+ "path/filepath"
"regexp"
"strings"
"sync"
@@ -267,6 +268,19 @@
// }
// ```
SymbolStyle SymbolStyle
+
+ // DirectoryFilters can be used to exclude unwanted directories from the
+ // workspace. By default, all directories are included. Filters are an
+ // operator, `+` to include and `-` to exclude, followed by a path prefix
+ // relative to the workspace folder. They are evaluated in order, and
+ // the last filter that applies to a path controls whether it is included.
+ // The path prefix can be empty, so an initial `-` excludes everything.
+ //
+ // Examples:
+ // Exclude node_modules: `-node_modules`
+ // Include only project_a: `-` (exclude everything), `+project_a`
+ // Include only project_a, but not node_modules inside it: `-`, `+project_a`, `-project_a/node_modules`
+ DirectoryFilters []string
}
// EnvSlice returns Env as a slice of k=v strings.
@@ -606,6 +620,7 @@
}
result.SetEnvSlice(o.EnvSlice())
result.BuildFlags = copySlice(o.BuildFlags)
+ result.DirectoryFilters = copySlice(o.DirectoryFilters)
copyAnalyzerMap := func(src map[string]Analyzer) map[string]Analyzer {
dst := make(map[string]Analyzer)
@@ -665,7 +680,22 @@
flags = append(flags, fmt.Sprintf("%s", flag))
}
o.BuildFlags = flags
-
+ case "directoryFilters":
+ ifilters, ok := value.([]interface{})
+ if !ok {
+ result.errorf("invalid type %T, expect list", value)
+ break
+ }
+ var filters []string
+ for _, ifilter := range ifilters {
+ filter := fmt.Sprint(ifilter)
+ if filter[0] != '+' && filter[0] != '-' {
+ result.errorf("invalid filter %q, must start with + or -", filter)
+ return result
+ }
+ filters = append(filters, filepath.FromSlash(filter))
+ }
+ o.DirectoryFilters = filters
case "completionDocumentation":
result.setBool(&o.CompletionDocumentation)
case "usePlaceholders":
