internal/lsp: limit concurrent file reads
We need to limit the concurrent file reads to prevent `too many open
file` errors. The first attempt to solve this was done in change 189437,
but this change turned out to be incorrect.
So this change reverts the changes made in change 189437 and instead
adds an new semaphore around the `nativeFileHandle.Read` method.
The limit is currently set to half of the lowest open file system limit
of *nix, Windows and MacOS system (their respective limits are 1024, 16384,
and 256).
Change-Id: I27d741e3a3c36eb11bfa7457575f5d5831c06ad7
Reviewed-on: https://go-review.googlesource.com/c/tools/+/190417
Reviewed-by: Ian Cottrell <iancottrell@google.com>
Run-TryBot: Ian Cottrell <iancottrell@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
diff --git a/internal/lsp/cache/external.go b/internal/lsp/cache/external.go
index b52899b..7a938a2 100644
--- a/internal/lsp/cache/external.go
+++ b/internal/lsp/cache/external.go
@@ -15,6 +15,9 @@
"golang.org/x/tools/internal/span"
)
+// ioLimit limits the number of parallel file reads per process.
+var ioLimit = make(chan struct{}, 128)
+
// nativeFileSystem implements FileSystem reading from the normal os file system.
type nativeFileSystem struct{}
@@ -54,6 +57,8 @@
func (h *nativeFileHandle) Read(ctx context.Context) ([]byte, string, error) {
ctx, done := trace.StartSpan(ctx, "cache.nativeFileHandle.Read", telemetry.File.Of(h.identity.URI.Filename()))
defer done()
+ ioLimit <- struct{}{}
+ defer func() { <-ioLimit }()
//TODO: this should fail if the version is not the same as the handle
data, err := ioutil.ReadFile(h.identity.URI.Filename())
if err != nil {
diff --git a/internal/lsp/cache/parse.go b/internal/lsp/cache/parse.go
index 3da4a6f..8e9def8 100644
--- a/internal/lsp/cache/parse.go
+++ b/internal/lsp/cache/parse.go
@@ -20,8 +20,8 @@
errors "golang.org/x/xerrors"
)
-// Limits the number of parallel file reads per process.
-var ioLimit = make(chan struct{}, 20)
+// Limits the number of parallel parser calls per process.
+var parseLimit = make(chan struct{}, 20)
// parseKey uniquely identifies a parsed Go file.
type parseKey struct {
@@ -104,13 +104,12 @@
ctx, done := trace.StartSpan(ctx, "cache.parseGo", telemetry.File.Of(fh.Identity().URI.Filename()))
defer done()
- ioLimit <- struct{}{}
buf, _, err := fh.Read(ctx)
- <-ioLimit
if err != nil {
return nil, err
}
-
+ parseLimit <- struct{}{}
+ defer func() { <-parseLimit }()
parserMode := parser.AllErrors | parser.ParseComments
if mode == source.ParseHeader {
parserMode = parser.ImportsOnly | parser.ParseComments
