gopls/internal/regtest: add benchmarks for completions
This change adds benchmarks for completions running against x/tools
codebase. We still maintain support for manually configured completion
benchmarks through the CLI; run instructions are added as comments in
code.
Change-Id: I3bcf7edac183397ad01755b767798925c4bd30ba
Reviewed-on: https://go-review.googlesource.com/c/tools/+/256877
Run-TryBot: Danish Dua <danishdua@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Heschi Kreinick <heschi@google.com>
Trust: Danish Dua <danishdua@google.com>
diff --git a/gopls/internal/regtest/bench_test.go b/gopls/internal/regtest/bench_test.go
index 0193968..8a7fbdb 100644
--- a/gopls/internal/regtest/bench_test.go
+++ b/gopls/internal/regtest/bench_test.go
@@ -12,142 +12,95 @@
"golang.org/x/tools/internal/lsp/protocol"
)
-var iwlBench = struct {
+func printBenchmarkResults(result testing.BenchmarkResult) {
+ fmt.Println("Benchmark Statistics:")
+ fmt.Println(result.String())
+ fmt.Println(result.MemString())
+}
+
+var iwlOptions struct {
workdir string
-}{}
+}
func init() {
- flag.StringVar(&iwlBench.workdir, "iwl_workdir", "", "if set, run IWL benchmark in this directory")
+ flag.StringVar(&iwlOptions.workdir, "iwl_workdir", "", "if set, run IWL benchmark in this directory")
}
func TestBenchmarkIWL(t *testing.T) {
- if iwlBench.workdir == "" {
+ if iwlOptions.workdir == "" {
t.Skip("-iwl_workdir not configured")
}
- opts := stressTestOptions(iwlBench.workdir)
+
+ opts := stressTestOptions(iwlOptions.workdir)
// Don't skip hooks, so that we can wait for IWL.
opts = append(opts, SkipHooks(false))
- b := testing.Benchmark(func(b *testing.B) {
+
+ results := testing.Benchmark(func(b *testing.B) {
for i := 0; i < b.N; i++ {
withOptions(opts...).run(t, "", func(t *testing.T, env *Env) {
env.Await(InitialWorkspaceLoad)
})
}
})
- printBench(b)
+
+ printBenchmarkResults(results)
}
-var symbolBench = struct {
+var symbolOptions struct {
workdir, query, matcher, style string
printResults bool
-}{}
+}
func init() {
- flag.StringVar(&symbolBench.workdir, "symbol_workdir", "", "if set, run symbol benchmark in this directory")
- flag.StringVar(&symbolBench.query, "symbol_query", "test", "symbol query to use in benchmark")
- flag.StringVar(&symbolBench.matcher, "symbol_matcher", "", "symbol matcher to use in benchmark")
- flag.StringVar(&symbolBench.style, "symbol_style", "", "symbol style to use in benchmark")
- flag.BoolVar(&symbolBench.printResults, "symbol_print_results", false, "whether to print symbol query results")
+ flag.StringVar(&symbolOptions.workdir, "symbol_workdir", "", "if set, run symbol benchmark in this directory")
+ flag.StringVar(&symbolOptions.query, "symbol_query", "test", "symbol query to use in benchmark")
+ flag.StringVar(&symbolOptions.matcher, "symbol_matcher", "", "symbol matcher to use in benchmark")
+ flag.StringVar(&symbolOptions.style, "symbol_style", "", "symbol style to use in benchmark")
+ flag.BoolVar(&symbolOptions.printResults, "symbol_print_results", false, "whether to print symbol query results")
}
func TestBenchmarkSymbols(t *testing.T) {
- if symbolBench.workdir == "" {
+ if symbolOptions.workdir == "" {
t.Skip("-symbol_workdir not configured")
}
- opts := stressTestOptions(symbolBench.workdir)
+
+ opts := stressTestOptions(symbolOptions.workdir)
conf := EditorConfig{}
- if symbolBench.matcher != "" {
- conf.SymbolMatcher = &symbolBench.matcher
+ if symbolOptions.matcher != "" {
+ conf.SymbolMatcher = &symbolOptions.matcher
}
- if symbolBench.style != "" {
- conf.SymbolStyle = &symbolBench.style
+ if symbolOptions.style != "" {
+ conf.SymbolStyle = &symbolOptions.style
}
opts = append(opts, conf)
+
withOptions(opts...).run(t, "", func(t *testing.T, env *Env) {
// We can't Await in this test, since we have disabled hooks. Instead, run
// one symbol request to completion to ensure all necessary cache entries
// are populated.
- results, err := env.Editor.Server.Symbol(env.Ctx, &protocol.WorkspaceSymbolParams{
- Query: symbolBench.query,
+ symbols, err := env.Editor.Server.Symbol(env.Ctx, &protocol.WorkspaceSymbolParams{
+ Query: symbolOptions.query,
})
if err != nil {
t.Fatal(err)
}
- if symbolBench.printResults {
+
+ if symbolOptions.printResults {
fmt.Println("Results:")
- for i := 0; i < len(results); i++ {
- fmt.Printf("\t%d. %s (%s)\n", i, results[i].Name, results[i].ContainerName)
+ for i := 0; i < len(symbols); i++ {
+ fmt.Printf("\t%d. %s (%s)\n", i, symbols[i].Name, symbols[i].ContainerName)
}
}
- b := testing.Benchmark(func(b *testing.B) {
+
+ results := testing.Benchmark(func(b *testing.B) {
for i := 0; i < b.N; i++ {
if _, err := env.Editor.Server.Symbol(env.Ctx, &protocol.WorkspaceSymbolParams{
- Query: symbolBench.query,
+ Query: symbolOptions.query,
}); err != nil {
t.Fatal(err)
}
}
})
- printBench(b)
- })
-}
-
-func printBench(b testing.BenchmarkResult) {
- fmt.Println("Benchmark stats:")
- fmt.Println(b.String())
- fmt.Println(b.MemString())
-}
-
-func dummyCompletionBenchmarkFunction() { const s = "placeholder"; fmt.Printf("%s", s) }
-
-var completionBench = struct {
- workdir, fileName, locationRegexp string
- printResults bool
-}{}
-
-func init() {
- flag.StringVar(&completionBench.workdir, "completion_workdir", "", "if set run completion benchmark in this directory (other benchmark flags expect an x/tools dir)")
- flag.StringVar(&completionBench.fileName, "completion_file", "internal/lsp/regtest/bench_test.go", "relative path to the file to complete")
- flag.StringVar(&completionBench.locationRegexp, "completion_regexp", `dummyCompletionBenchmarkFunction.*fmt\.Printf\("%s", s(\))`, "regexp location to complete at")
- flag.BoolVar(&completionBench.printResults, "completion_print_results", false, "whether to print completion results")
-}
-
-func TestBenchmarkCompletion(t *testing.T) {
- if completionBench.workdir == "" {
- t.Skip("-completion_workdir not configured")
- }
- opts := stressTestOptions(completionBench.workdir)
- // Completion gives bad results if IWL is not yet complete, so we must await
- // it first (and therefore need hooks).
- opts = append(opts, SkipHooks(false))
- withOptions(opts...).run(t, "", func(t *testing.T, env *Env) {
- env.Await(InitialWorkspaceLoad)
- env.OpenFile(completionBench.fileName)
- params := &protocol.CompletionParams{}
- params.Context.TriggerCharacter = "s"
- params.Context.TriggerKind = protocol.TriggerCharacter
- params.TextDocument.URI = env.Sandbox.Workdir.URI(completionBench.fileName)
- params.Position = env.RegexpSearch(completionBench.fileName, completionBench.locationRegexp).ToProtocolPosition()
-
- // Run one completion to make sure everything is warm.
- list, err := env.Editor.Server.Completion(env.Ctx, params)
- if err != nil {
- t.Fatal(err)
- }
- if completionBench.printResults {
- fmt.Println("Results:")
- for i := 0; i < len(list.Items); i++ {
- fmt.Printf("\t%d. %v\n", i, list.Items[i])
- }
- }
- b := testing.Benchmark(func(b *testing.B) {
- for i := 0; i < b.N; i++ {
- _, err := env.Editor.Server.Completion(env.Ctx, params)
- if err != nil {
- t.Fatal(err)
- }
- }
- })
- printBench(b)
+ printBenchmarkResults(results)
})
}
diff --git a/gopls/internal/regtest/completion_bench_test.go b/gopls/internal/regtest/completion_bench_test.go
new file mode 100644
index 0000000..1b6fe46
--- /dev/null
+++ b/gopls/internal/regtest/completion_bench_test.go
@@ -0,0 +1,201 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package regtest
+
+import (
+ "flag"
+ "fmt"
+ "runtime"
+ "strings"
+ "testing"
+
+ "golang.org/x/tools/internal/lsp/fake"
+)
+
+// dummyCompletionFunction to test manually configured completion using CLI.
+func dummyCompletionFunction() { const s = "placeholder"; fmt.Printf("%s", s) }
+
+type completionBenchOptions struct {
+ workdir, file, locationRegexp string
+ printResults bool
+ // hook to run edits before initial completion, not supported for manually
+ // configured completions.
+ preCompletionEdits func(*Env)
+}
+
+var completionOptions = completionBenchOptions{}
+
+func init() {
+ flag.StringVar(&completionOptions.workdir, "completion_workdir", "", "directory to run completion benchmarks in")
+ flag.StringVar(&completionOptions.file, "completion_file", "", "relative path to the file to complete in")
+ flag.StringVar(&completionOptions.locationRegexp, "completion_regexp", "", "regexp location to complete at")
+ flag.BoolVar(&completionOptions.printResults, "completion_print_results", false, "whether to print completion results")
+}
+
+func benchmarkCompletion(options completionBenchOptions, t *testing.T) {
+ if completionOptions.workdir == "" {
+ t.Skip("-completion_workdir not configured, skipping benchmark")
+ }
+
+ opts := stressTestOptions(options.workdir)
+
+ // Completion gives bad results if IWL is not yet complete, so we must await
+ // it first (and therefore need hooks).
+ opts = append(opts, SkipHooks(false))
+
+ withOptions(opts...).run(t, "", func(t *testing.T, env *Env) {
+ env.Await(InitialWorkspaceLoad)
+ env.OpenFile(options.file)
+
+ // Run edits required for this completion.
+ if options.preCompletionEdits != nil {
+ options.preCompletionEdits(env)
+ }
+
+ // Add a comment as a marker at the start of the file, we'll replace
+ // this in every iteration to trigger type checking and hence emulate
+ // a more real world scenario.
+ env.EditBuffer(options.file, fake.Edit{Text: "// 0\n"})
+
+ // Run a completion to make sure the system is warm.
+ pos := env.RegexpSearch(options.file, options.locationRegexp)
+ completions := env.Completion(options.file, pos)
+
+ if options.printResults {
+ fmt.Println("Results:")
+ for i := 0; i < len(completions.Items); i++ {
+ fmt.Printf("\t%d. %v\n", i, completions.Items[i])
+ }
+ }
+
+ results := testing.Benchmark(func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ b.StopTimer()
+ env.RegexpReplace(options.file, `\/\/ \d*`, fmt.Sprintf("// %d", i))
+
+ // explicitly garbage collect since we don't want to count this
+ // time in completion benchmarks.
+ if i%10 == 0 {
+ runtime.GC()
+ }
+ b.StartTimer()
+
+ env.Completion(options.file, pos)
+ }
+ })
+
+ printBenchmarkResults(results)
+ })
+}
+
+// endPosInBuffer returns the position for last character in the buffer for
+// the given file.
+func endPosInBuffer(env *Env, name string) fake.Pos {
+ buffer := env.Editor.BufferText(name)
+ lines := strings.Split(buffer, "\n")
+ numLines := len(lines)
+
+ return fake.Pos{
+ Line: numLines - 1,
+ Column: len([]rune(lines[numLines-1])),
+ }
+}
+
+// Benchmark completion at a specified file and location. When no CLI options
+// are specified, this test is skipped.
+// To Run (from x/tools/gopls) against the dummy function above:
+// go test -v ./internal/regtest -run=TestBenchmarkConfiguredCompletion
+// -completion_workdir="$HOME/Developer/tools"
+// -completion_file="gopls/internal/regtest/completion_bench_test.go"
+// -completion_regexp="dummyCompletionFunction.*fmt\.Printf\(\"%s\", s(\))"
+func TestBenchmarkConfiguredCompletion(t *testing.T) {
+ benchmarkCompletion(completionOptions, t)
+}
+
+// To run (from x/tools/gopls):
+// go test -v ./internal/regtest -run TestBenchmark<>Completion
+// -completion_workdir="$HOME/Developer/tools"
+// where <> is one of the tests below. completion_workdir should be path to
+// x/tools on your system.
+
+// Benchmark struct completion in tools codebase.
+func TestBenchmarkStructCompletion(t *testing.T) {
+ file := "internal/lsp/cache/session.go"
+
+ preCompletionEdits := func(env *Env) {
+ env.OpenFile(file)
+ originalBuffer := env.Editor.BufferText(file)
+ env.EditBuffer(file, fake.Edit{
+ End: endPosInBuffer(env, file),
+ Text: originalBuffer + "\nvar testVariable map[string]bool = Session{}.\n",
+ })
+ }
+
+ benchmarkCompletion(completionBenchOptions{
+ workdir: completionOptions.workdir,
+ file: file,
+ locationRegexp: `var testVariable map\[string\]bool = Session{}(\.)`,
+ preCompletionEdits: preCompletionEdits,
+ printResults: completionOptions.printResults,
+ }, t)
+}
+
+// Benchmark import completion in tools codebase.
+func TestBenchmarkImportCompletion(t *testing.T) {
+ benchmarkCompletion(completionBenchOptions{
+ workdir: completionOptions.workdir,
+ file: "internal/lsp/source/completion/completion.go",
+ locationRegexp: `go\/()`,
+ printResults: completionOptions.printResults,
+ }, t)
+}
+
+// Benchmark slice completion in tools codebase.
+func TestBenchmarkSliceCompletion(t *testing.T) {
+ file := "internal/lsp/cache/session.go"
+
+ preCompletionEdits := func(env *Env) {
+ env.OpenFile(file)
+ originalBuffer := env.Editor.BufferText(file)
+ env.EditBuffer(file, fake.Edit{
+ End: endPosInBuffer(env, file),
+ Text: originalBuffer + "\nvar testVariable []byte = \n",
+ })
+ }
+
+ benchmarkCompletion(completionBenchOptions{
+ workdir: completionOptions.workdir,
+ file: file,
+ locationRegexp: `var testVariable \[\]byte (=)`,
+ preCompletionEdits: preCompletionEdits,
+ printResults: completionOptions.printResults,
+ }, t)
+}
+
+// Benchmark deep completion in function call in tools codebase.
+func TestBenchmarkFuncDeepCompletion(t *testing.T) {
+ file := "internal/lsp/source/completion/completion.go"
+ fileContent := `
+func (c *completer) _() {
+ c.inference.kindMatches(c.)
+}
+`
+ preCompletionEdits := func(env *Env) {
+ env.OpenFile(file)
+ originalBuffer := env.Editor.BufferText(file)
+ env.EditBuffer(file, fake.Edit{
+ End: endPosInBuffer(env, file),
+ Text: originalBuffer + fileContent,
+ })
+ }
+
+ benchmarkCompletion(completionBenchOptions{
+ workdir: completionOptions.workdir,
+ file: file,
+ locationRegexp: `func \(c \*completer\) _\(\) {\n\tc\.inference\.kindMatches\((c)`,
+ preCompletionEdits: preCompletionEdits,
+ printResults: completionOptions.printResults,
+ }, t)
+}
diff --git a/gopls/internal/regtest/completion_test.go b/gopls/internal/regtest/completion_test.go
index 7cc0b6a..2651f59 100644
--- a/gopls/internal/regtest/completion_test.go
+++ b/gopls/internal/regtest/completion_test.go
@@ -131,10 +131,8 @@
)
}
env.OpenFile(tc.filename)
- completions, err := env.Editor.Completion(env.Ctx, tc.filename, env.RegexpSearch(tc.filename, tc.triggerRegexp))
- if err != nil {
- t.Fatal(err)
- }
+ completions := env.Completion(tc.filename, env.RegexpSearch(tc.filename, tc.triggerRegexp))
+
// Check that the completion item suggestions are in the range
// of the file.
lineCount := len(strings.Split(env.Editor.BufferText(tc.filename), "\n"))
@@ -183,13 +181,10 @@
want := []string{"ma", "ma_test", "main", "math", "math_test"}
run(t, files, func(t *testing.T, env *Env) {
env.OpenFile("math/add.go")
- completions, err := env.Editor.Completion(env.Ctx, "math/add.go", fake.Pos{
+ completions := env.Completion("math/add.go", fake.Pos{
Line: 0,
Column: 10,
})
- if err != nil {
- t.Fatal(err)
- }
diff := compareCompletionResults(want, completions.Items)
if diff != "" {
diff --git a/gopls/internal/regtest/wrappers.go b/gopls/internal/regtest/wrappers.go
index d00712a..8d8136f 100644
--- a/gopls/internal/regtest/wrappers.go
+++ b/gopls/internal/regtest/wrappers.go
@@ -270,6 +270,16 @@
return locations
}
+// Completion executes a completion request on the server.
+func (e *Env) Completion(path string, pos fake.Pos) *protocol.CompletionList {
+ e.T.Helper()
+ completions, err := e.Editor.Completion(e.Ctx, path, pos)
+ if err != nil {
+ e.T.Fatal(err)
+ }
+ return completions
+}
+
// CodeAction calls testDocument/codeAction for the given path, and calls
// t.Fatal if there are errors.
func (e *Env) CodeAction(path string) []protocol.CodeAction {
diff --git a/internal/lsp/fake/editor.go b/internal/lsp/fake/editor.go
index f9a5743..38546af 100644
--- a/internal/lsp/fake/editor.go
+++ b/internal/lsp/fake/editor.go
@@ -178,6 +178,7 @@
"verboseWorkDoneProgress": true,
"env": e.overlayEnv(),
"expandWorkspaceToModule": !e.Config.LimitWorkspaceScope,
+ "completionBudget": "10s",
}
if e.Config.CodeLens != nil {
