internal/lsp/source: improve completions at file scope

Add support for var/func/const/type/import keywords at the file scope.
I left out "package" because, currently, if you are completing
something that means you must already have a package declaration. The
main hurdle was that anything other than a decl keyword shows up as
an *ast.BadDecl at the file scope. To properly detect the prefix we
manually scan for the token containing the position.

I also made a couple small drive-by improvements:
 - Also suggest "goto" and "type" keywords in functions.
 - Allow completing directly before a comment, e.g. "foo<>//". I
   needed this for a test that would have been annoying to write
   otherwise.

Updates golang/go#34009.

Change-Id: I290e7bdda9e66a16f996cdc291985a54bf375231
Reviewed-on: https://go-review.googlesource.com/c/tools/+/217500
Run-TryBot: Rebecca Stambler <rstambler@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
diff --git a/internal/lsp/source/completion.go b/internal/lsp/source/completion.go
index 8fc3930..bd25052 100644
--- a/internal/lsp/source/completion.go
+++ b/internal/lsp/source/completion.go
@@ -493,16 +493,24 @@
 	}
 
 	// Detect our surrounding identifier.
-	if ident, ok := path[0].(*ast.Ident); ok {
+	switch leaf := path[0].(type) {
+	case *ast.Ident:
 		// In the normal case, our leaf AST node is the identifer being completed.
-		c.setSurrounding(ident)
-	} else {
+		c.setSurrounding(leaf)
+	case *ast.BadDecl:
+		// You don't get *ast.Idents at the file level, so look for bad
+		// decls and manually extract the surrounding token.
+		pos, _, lit := c.scanToken(ctx, src)
+		if pos.IsValid() {
+			c.setSurrounding(&ast.Ident{Name: lit, NamePos: pos})
+		}
+	default:
 		// Otherwise, manually extract the prefix if our containing token
 		// is a keyword. This improves completion after an "accidental
 		// keyword", e.g. completing to "variance" in "someFunc(var<>)".
-		id := scanKeyword(c.pos, c.snapshot.View().Session().Cache().FileSet().File(c.pos), src)
-		if id != nil {
-			c.setSurrounding(id)
+		pos, tkn, lit := c.scanToken(ctx, src)
+		if pos.IsValid() && tkn.IsKeyword() {
+			c.setSurrounding(&ast.Ident{Name: lit, NamePos: pos})
 		}
 	}
 
@@ -512,7 +520,7 @@
 
 	// If we're inside a comment return comment completions
 	for _, comment := range file.Comments {
-		if comment.Pos() <= rng.Start && rng.Start <= comment.End() {
+		if comment.Pos() < rng.Start && rng.Start <= comment.End() {
 			c.populateCommentCompletions(comment)
 			return c.items, c.getSurrounding(), nil
 		}
@@ -556,10 +564,6 @@
 		if err := c.lexical(); err != nil {
 			return nil, nil, err
 		}
-		if err := c.keyword(); err != nil {
-			return nil, nil, err
-		}
-
 	// The function name hasn't been typed yet, but the parens are there:
 	//   recv.‸(arg)
 	case *ast.TypeAssertExpr:
@@ -573,6 +577,10 @@
 			return nil, nil, err
 		}
 
+	// At the file scope, only keywords are allowed.
+	case *ast.BadDecl, *ast.File:
+		c.addKeywordCompletions()
+
 	default:
 		// fallback to lexical completions
 		if err := c.lexical(); err != nil {
@@ -583,20 +591,20 @@
 	return c.items, c.getSurrounding(), nil
 }
 
-// scanKeyword scans file for the ident or keyword containing or abutting pos.
-func scanKeyword(pos token.Pos, file *token.File, data []byte) *ast.Ident {
+// scanToken scans pgh's contents for the token containing pos.
+func (c *completer) scanToken(ctx context.Context, contents []byte) (token.Pos, token.Token, string) {
+	tok := c.snapshot.View().Session().Cache().FileSet().File(c.pos)
+
 	var s scanner.Scanner
-	s.Init(file, data, nil, 0)
+	s.Init(tok, contents, nil, 0)
 	for {
 		tknPos, tkn, lit := s.Scan()
-		if tkn == token.EOF || tknPos >= pos {
-			return nil
+		if tkn == token.EOF || tknPos >= c.pos {
+			return token.NoPos, token.ILLEGAL, ""
 		}
 
-		if tkn.IsKeyword() {
-			if tknPos <= pos && pos <= tknPos+token.Pos(len(lit)) {
-				return &ast.Ident{NamePos: tknPos, Name: lit}
-			}
+		if len(lit) > 0 && tknPos <= c.pos && c.pos <= tknPos+token.Pos(len(lit)) {
+			return tknPos, tkn, lit
 		}
 	}
 }
@@ -1002,6 +1010,9 @@
 		}
 	}
 
+	// Add keyword completion items appropriate in the current context.
+	c.addKeywordCompletions()
+
 	return nil
 }
 
diff --git a/internal/lsp/source/completion_keywords.go b/internal/lsp/source/completion_keywords.go
index dcb5c95..4cb7117 100644
--- a/internal/lsp/source/completion_keywords.go
+++ b/internal/lsp/source/completion_keywords.go
@@ -4,8 +4,6 @@
 	"go/ast"
 
 	"golang.org/x/tools/internal/lsp/protocol"
-
-	errors "golang.org/x/xerrors"
 )
 
 const (
@@ -36,22 +34,48 @@
 	VAR         = "var"
 )
 
-// keyword looks at the current scope of an *ast.Ident and recommends keywords
-func (c *completer) keyword() error {
-	keywordScore := float64(0.9)
-	if _, ok := c.path[0].(*ast.Ident); !ok {
-		// TODO(golang/go#34009): Support keyword completion in any context
-		return errors.Errorf("keywords are currently only recommended for identifiers")
-	}
-	// Track which keywords we've already determined are in a valid scope
-	// Use score to order keywords by how close we are to where they are useful
-	valid := make(map[string]float64)
+// addKeywordCompletions offers keyword candidates appropriate at the position.
+func (c *completer) addKeywordCompletions() {
+	const keywordScore = 0.9
 
-	// only suggest keywords at the begnning of a statement
+	seen := make(map[string]bool)
+
+	// addKeywords dedupes and adds completion items for the specified
+	// keywords with the specified score.
+	addKeywords := func(score float64, kws ...string) {
+		for _, kw := range kws {
+			if seen[kw] {
+				continue
+			}
+			seen[kw] = true
+
+			if c.matcher.Score(kw) > 0 {
+				c.items = append(c.items, CompletionItem{
+					Label:      kw,
+					Kind:       protocol.KeywordCompletion,
+					InsertText: kw,
+					Score:      score,
+				})
+			}
+		}
+	}
+
+	// If we are at the file scope, only offer decl keywords. We don't
+	// get *ast.Idents at the file scope because non-keyword identifiers
+	// turn into *ast.BadDecl, not *ast.Ident.
+	if len(c.path) == 1 || isASTFile(c.path[1]) {
+		addKeywords(keywordScore, TYPE, CONST, VAR, FUNC, IMPORT)
+		return
+	} else if _, ok := c.path[0].(*ast.Ident); !ok {
+		// Otherwise only offer keywords if the client is completing an identifier.
+		return
+	}
+
+	// Only suggest keywords if we are beginning a statement.
 	switch c.path[1].(type) {
 	case *ast.BlockStmt, *ast.CommClause, *ast.CaseClause, *ast.ExprStmt:
 	default:
-		return nil
+		return
 	}
 
 	// Filter out keywords depending on scope
@@ -62,7 +86,7 @@
 		case *ast.CaseClause:
 			// only recommend "fallthrough" and "break" within the bodies of a case clause
 			if c.pos > node.Colon {
-				valid[BREAK] = keywordScore
+				addKeywords(keywordScore, BREAK)
 				// "fallthrough" is only valid in switch statements.
 				// A case clause is always nested within a block statement in a switch statement,
 				// that block statement is nested within either a TypeSwitchStmt or a SwitchStmt.
@@ -70,45 +94,23 @@
 					continue
 				}
 				if _, ok := path[i+2].(*ast.SwitchStmt); ok {
-					valid[FALLTHROUGH] = keywordScore
+					addKeywords(keywordScore, FALLTHROUGH)
 				}
 			}
 		case *ast.CommClause:
 			if c.pos > node.Colon {
-				valid[BREAK] = keywordScore
+				addKeywords(keywordScore, BREAK)
 			}
 		case *ast.TypeSwitchStmt, *ast.SelectStmt, *ast.SwitchStmt:
-			valid[CASE] = keywordScore + lowScore
-			valid[DEFAULT] = keywordScore + lowScore
+			addKeywords(keywordScore+lowScore, CASE, DEFAULT)
 		case *ast.ForStmt:
-			valid[BREAK] = keywordScore
-			valid[CONTINUE] = keywordScore
+			addKeywords(keywordScore, BREAK, CONTINUE)
 		// This is a bit weak, functions allow for many keywords
 		case *ast.FuncDecl:
 			if node.Body != nil && c.pos > node.Body.Lbrace {
-				valid[DEFER] = keywordScore - lowScore
-				valid[RETURN] = keywordScore - lowScore
-				valid[FOR] = keywordScore - lowScore
-				valid[GO] = keywordScore - lowScore
-				valid[SWITCH] = keywordScore - lowScore
-				valid[SELECT] = keywordScore - lowScore
-				valid[IF] = keywordScore - lowScore
-				valid[ELSE] = keywordScore - lowScore
-				valid[VAR] = keywordScore - lowScore
-				valid[CONST] = keywordScore - lowScore
+				addKeywords(keywordScore-lowScore, DEFER, RETURN, FOR, GO, SWITCH, SELECT, IF, ELSE, VAR, CONST, GOTO, TYPE)
 			}
 		}
 	}
 
-	for ident, score := range valid {
-		if c.matcher.Score(ident) > 0 {
-			c.items = append(c.items, CompletionItem{
-				Label:      ident,
-				Kind:       protocol.KeywordCompletion,
-				InsertText: ident,
-				Score:      score,
-			})
-		}
-	}
-	return nil
 }
diff --git a/internal/lsp/source/util.go b/internal/lsp/source/util.go
index ffb166f..0a866bd 100644
--- a/internal/lsp/source/util.go
+++ b/internal/lsp/source/util.go
@@ -420,6 +420,11 @@
 	return ok
 }
 
+func isASTFile(n ast.Node) bool {
+	_, ok := n.(*ast.File)
+	return ok
+}
+
 // isSelector returns the enclosing *ast.SelectorExpr when pos is in the
 // selector.
 func enclosingSelector(path []ast.Node, pos token.Pos) *ast.SelectorExpr {
diff --git a/internal/lsp/testdata/lsp/primarymod/foo/foo.go b/internal/lsp/testdata/lsp/primarymod/foo/foo.go
index 277ac53..20ea183 100644
--- a/internal/lsp/testdata/lsp/primarymod/foo/foo.go
+++ b/internal/lsp/testdata/lsp/primarymod/foo/foo.go
@@ -27,4 +27,4 @@
 	}
 }
 
-type IntFoo int //@item(IntFoo, "IntFoo", "int", "type"),complete("", Foo, IntFoo, StructFoo)
+type IntFoo int //@item(IntFoo, "IntFoo", "int", "type")
diff --git a/internal/lsp/testdata/lsp/primarymod/keywords/keywords.go b/internal/lsp/testdata/lsp/primarymod/keywords/keywords.go
index 5fda68d..7e7fd5b 100644
--- a/internal/lsp/testdata/lsp/primarymod/keywords/keywords.go
+++ b/internal/lsp/testdata/lsp/primarymod/keywords/keywords.go
@@ -1,5 +1,7 @@
 package keywords
 
+//@rank("", type),rank("", func),rank("", var),rank("", const),rank("", import)
+
 func _() {
 	var test int
 	var tChan chan int
@@ -42,7 +44,7 @@
 
 	f //@complete(" //", for)
 	d //@complete(" //", defer)
-	g //@complete(" //", go)
+	g //@rank(" //", go),rank(" //", goto)
 	r //@complete(" //", return)
 	i //@complete(" //", if)
 	e //@complete(" //", else)
@@ -72,3 +74,4 @@
 /* return */ //@item(return, "return", "", "keyword")
 /* var */ //@item(var, "var", "", "keyword")
 /* const */ //@item(const, "const", "", "keyword")
+/* goto */ //@item(goto, "goto", "", "keyword")
diff --git a/internal/lsp/testdata/lsp/summary.txt.golden b/internal/lsp/testdata/lsp/summary.txt.golden
index 4845142..b36331b 100644
--- a/internal/lsp/testdata/lsp/summary.txt.golden
+++ b/internal/lsp/testdata/lsp/summary.txt.golden
@@ -1,11 +1,11 @@
 -- summary --
 CodeLensCount = 0
-CompletionsCount = 228
+CompletionsCount = 226
 CompletionSnippetCount = 67
 UnimportedCompletionsCount = 11
 DeepCompletionsCount = 5
 FuzzyCompletionsCount = 8
-RankedCompletionsCount = 102
+RankedCompletionsCount = 109
 CaseSensitiveCompletionsCount = 4
 DiagnosticsCount = 38
 FoldingRangesCount = 2