internal/lsp: add ast fields to comment completion for declarations

* adds support for comment completion inside declarations
* improves scoring for completion results for comments
* adds comment completion support for non-exported symbols
* adds pruning for results that don't match text surrounding cursor
* tests for comment completion

Change-Id: Icb445a469cee3122fe032630bee037c7bdfe2e18
Reviewed-on: https://go-review.googlesource.com/c/tools/+/249639
Run-TryBot: Danish Dua <danishdua@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Heschi Kreinick <heschi@google.com>
diff --git a/internal/lsp/source/completion.go b/internal/lsp/source/completion.go
index 670865f..36380db 100644
--- a/internal/lsp/source/completion.go
+++ b/internal/lsp/source/completion.go
@@ -559,6 +559,9 @@
 	// If we're inside a comment return comment completions
 	for _, comment := range pgf.File.Comments {
 		if comment.Pos() < rng.Start && rng.Start <= comment.End() {
+			// deep completion doesn't work properly in comments since we don't
+			// have a type object to complete further
+			c.deepState.maxDepth = 0
 			c.populateCommentCompletions(ctx, comment)
 			return c.items, c.getSurrounding(), nil
 		}
@@ -721,8 +724,7 @@
 	}
 }
 
-// populateCommentCompletions yields completions for exported
-// symbols immediately preceding comment.
+// populateCommentCompletions yields completions for comments preceding or in declarationss
 func (c *completer) populateCommentCompletions(ctx context.Context, comment *ast.CommentGroup) {
 	// Using the comment position find the line after
 	file := c.snapshot.FileSet().File(comment.End())
@@ -730,22 +732,18 @@
 		return
 	}
 
-	line := file.Line(comment.End())
-	if file.LineCount() < line+1 {
-		return
-	}
-
-	nextLinePos := file.LineStart(line + 1)
-	if !nextLinePos.IsValid() {
-		return
-	}
+	commentLine := file.Line(comment.End())
 
 	// comment is valid, set surrounding as word boundaries around cursor
 	c.setSurroundingForComment(comment)
+	cursorText := c.surrounding.content
 
 	// Using the next line pos, grab and parse the exported symbol on that line
 	for _, n := range c.file.Decls {
-		if n.Pos() != nextLinePos {
+		declLine := file.Line(n.Pos())
+		// if the comment is not in, directly above or on the same line as a declaration
+		if declLine != commentLine && declLine != commentLine+1 &&
+			!(n.Pos() <= comment.Pos() && comment.End() <= n.End()) {
 			continue
 		}
 		switch node := n.(type) {
@@ -755,23 +753,85 @@
 				switch spec := spec.(type) {
 				case *ast.ValueSpec:
 					for _, name := range spec.Names {
-						if name.String() == "_" || !name.IsExported() {
+						if name.String() == "_" || !strings.HasPrefix(name.String(), cursorText) {
 							continue
 						}
 						obj := c.pkg.GetTypesInfo().ObjectOf(name)
 						c.found(ctx, candidate{obj: obj, score: stdScore})
 					}
 				case *ast.TypeSpec:
-					if spec.Name.String() == "_" || !spec.Name.IsExported() {
+					// add TypeSpec fields to completion
+					switch typeNode := spec.Type.(type) {
+					case *ast.StructType:
+						c.addFieldItems(ctx, typeNode.Fields)
+					case *ast.FuncType:
+						c.addFieldItems(ctx, typeNode.Params)
+						c.addFieldItems(ctx, typeNode.Results)
+					case *ast.InterfaceType:
+						c.addFieldItems(ctx, typeNode.Methods)
+					}
+
+					if spec.Name.String() == "_" || !strings.HasPrefix(spec.Name.String(), cursorText) {
 						continue
 					}
+
 					obj := c.pkg.GetTypesInfo().ObjectOf(spec.Name)
-					c.found(ctx, candidate{obj: obj, score: stdScore})
+					// Type name should get a higher score than fields but not highScore by default
+					// since field near a comment cursor gets a highScore
+					score := stdScore * 1.1
+					// If type declaration is on the line after comment, give it a highScore.
+					if declLine == commentLine+1 {
+						score = highScore
+					}
+
+					// we use c.item in addFieldItems so we have to use c.item here to ensure scoring
+					// order is maintained. c.found manipulates the score
+					if item, err := c.item(ctx, candidate{obj: obj, name: obj.Name(), score: score}); err == nil {
+						c.items = append(c.items, item)
+					}
 				}
 			}
 		// handle functions
 		case *ast.FuncDecl:
-			if node.Name.String() == "_" || !node.Name.IsExported() {
+			c.addFieldItems(ctx, node.Recv)
+			c.addFieldItems(ctx, node.Type.Params)
+			c.addFieldItems(ctx, node.Type.Results)
+
+			// collect receiver struct fields
+			if node.Recv != nil {
+				for _, fields := range node.Recv.List {
+					for _, name := range fields.Names {
+						obj := c.pkg.GetTypesInfo().ObjectOf(name)
+						if obj == nil {
+							continue
+						}
+
+						recvType := obj.Type().Underlying()
+						if ptr, ok := recvType.(*types.Pointer); ok {
+							recvType = ptr.Elem()
+						}
+						recvStruct, ok := recvType.Underlying().(*types.Struct)
+						if !ok {
+							continue
+						}
+						for i := 0; i < recvStruct.NumFields(); i++ {
+							field := recvStruct.Field(i)
+							if !strings.HasPrefix(field.Name(), cursorText) {
+								continue
+							}
+							// we use c.item in addFieldItems so we have to use c.item here to ensure scoring
+							// order is maintained. c.found maniplulates the score
+							item, err := c.item(ctx, candidate{obj: field, name: field.Name(), score: lowScore})
+							if err != nil {
+								continue
+							}
+							c.items = append(c.items, item)
+						}
+					}
+				}
+			}
+
+			if node.Name.String() == "_" || !strings.HasPrefix(node.Name.String(), cursorText) {
 				continue
 			}
 
@@ -780,13 +840,13 @@
 				continue
 			}
 
-			// We don't want expandFuncCall inside comments. We add this directly to the
-			// completions list because using c.found sets expandFuncCall to true by default
+			// We don't want to expandFuncCall inside comments.
+			// c.found() doesn't respect this setting
 			item, err := c.item(ctx, candidate{
 				obj:            obj,
 				name:           obj.Name(),
 				expandFuncCall: false,
-				score:          stdScore,
+				score:          highScore,
 			})
 			if err != nil {
 				continue
@@ -835,6 +895,45 @@
 	return unicode.In(charRune, unicode.Letter, unicode.Digit) || char == '_'
 }
 
+// adds struct fields, interface methods, function declaration fields to completion
+func (c *completer) addFieldItems(ctx context.Context, fields *ast.FieldList) {
+	if fields == nil {
+		return
+	}
+
+	cursor := c.surrounding.cursor
+	surroundingPrefix := c.surrounding.content
+	for _, field := range fields.List {
+		for _, name := range field.Names {
+			if name.String() == "_" ||
+				!strings.HasPrefix(name.String(), surroundingPrefix) {
+				continue
+			}
+			obj := c.pkg.GetTypesInfo().ObjectOf(name)
+
+			// if we're in a field comment/doc, score that field as more relevant
+			score := stdScore
+			if field.Comment != nil && field.Comment.Pos() <= cursor && cursor <= field.Comment.End() {
+				score = highScore
+			} else if field.Doc != nil && field.Doc.Pos() <= cursor && cursor <= field.Doc.End() {
+				score = highScore
+			}
+
+			cand := candidate{
+				obj:            obj,
+				name:           obj.Name(),
+				expandFuncCall: false,
+				score:          score,
+			}
+			// We don't want to expandFuncCall inside comments.
+			// c.found() doesn't respect this setting
+			if item, err := c.item(ctx, cand); err == nil {
+				c.items = append(c.items, item)
+			}
+		}
+	}
+}
+
 func (c *completer) wantStructFieldCompletions() bool {
 	clInfo := c.enclosingCompositeLiteral
 	if clInfo == nil {
diff --git a/internal/lsp/testdata/lsp/primarymod/comment_completion/comment_completion.go.in b/internal/lsp/testdata/lsp/primarymod/comment_completion/comment_completion.go.in
new file mode 100644
index 0000000..e48b581
--- /dev/null
+++ b/internal/lsp/testdata/lsp/primarymod/comment_completion/comment_completion.go.in
@@ -0,0 +1,70 @@
+package comment_completion
+
+var p bool
+
+//@complete(re"$")
+
+func _() {
+	var a int
+
+	switch a {
+	case 1:
+		//@complete(re"$")
+		_ = a
+	}
+
+	var b chan int
+	select {
+	case <-b:
+		//@complete(re"$")
+		_ = b
+	}
+
+	var (
+		//@complete(re"$")
+		_ = a
+	)
+}
+
+// //@complete(" ", variableC)
+var C string //@item(variableC, "C", "string", "var") //@complete(" ", variableC)
+
+// //@complete(" ", constant)
+const Constant = "example" //@item(constant, "Constant", "string", "const") //@complete(" ", constant)
+
+// //@complete(" ", structType, fieldA, fieldB)
+type StructType struct { //@item(structType, "StructType", "struct{...}", "struct") //@complete(" ", structType, fieldA, fieldB)
+	// //@complete(" ", fieldA, structType, fieldB)
+	A string //@item(fieldA, "A", "string", "field") //@complete(" ", fieldA, structType, fieldB)
+	b int    //@item(fieldB, "b", "int", "field") //@complete(" ", fieldB, structType, fieldA)
+}
+
+// //@complete(" ", method, paramX, resultY, structRecv, fieldA, fieldB)
+func (structType *StructType) Method(X int) (Y int) { //@item(structRecv, "structType", "*StructType", "var"),item(method, "Method", "func(X int) (Y int)", "method"),item(paramX, "X", "int", "var"),item(resultY, "Y", "int", "var")
+	// //@complete(" ", method, paramX, resultY, structRecv, fieldA, fieldB)
+	return
+}
+
+// //@complete(" ", newType)
+type NewType string //@item(newType, "NewType", "string", "type") //@complete(" ", newType)
+
+// //@complete(" ", testInterface, testA, testB)
+type TestInterface interface { //@item(testInterface, "TestInterface", "interface{...}", "interface")
+	// //@complete(" ", testA, testInterface, testB)
+	TestA(L string) (M int) //@item(testA, "TestA", "func(L string) (M int)", "method"),item(paramL, "L", "var", "string"),item(resM, "M", "var", "int") //@complete(" ", testA, testInterface, testB)
+	TestB(N int) bool       //@item(testB, "TestB", "func(N int) bool", "method"),item(paramN, "N", "var", "int") //@complete(" ", testB, testInterface, testA)
+}
+
+// //@complete(" ", function)
+func Function() int { //@item(function, "Function", "func() int", "func") //@complete(" ", function)
+	// //@complete(" ", function)
+	return 0
+}
+
+// This tests multiline block comments and completion with prefix
+// Lorem Ipsum Multili//@complete("Multi", multiline)
+// Lorem ipsum dolor sit ametom
+func Multiline() int { //@item(multiline, "Multiline", "func() int", "func")
+	// //@complete(" ", multiline)
+	return 0
+}
diff --git a/internal/lsp/testdata/lsp/primarymod/comments/comments.go b/internal/lsp/testdata/lsp/primarymod/comments/comments.go
deleted file mode 100644
index e261cfd..0000000
--- a/internal/lsp/testdata/lsp/primarymod/comments/comments.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package comments
-
-var p bool
-
-//@complete(re"$")
-
-func _() {
-	var a int
-
-	switch a {
-	case 1:
-		//@complete(re"$")
-		_ = a
-	}
-
-	var b chan int
-	select {
-	case <-b:
-		//@complete(re"$")
-		_ = b
-	}
-
-	var (
-		//@complete(re"$")
-		_ = a
-	)
-}
diff --git a/internal/lsp/testdata/lsp/primarymod/complit/complit.go.in b/internal/lsp/testdata/lsp/primarymod/complit/complit.go.in
index 465a72c..c888c01 100644
--- a/internal/lsp/testdata/lsp/primarymod/complit/complit.go.in
+++ b/internal/lsp/testdata/lsp/primarymod/complit/complit.go.in
@@ -1,29 +1,5 @@
 package complit
 
-// exported comment completions
-
-// //@complete(" ", cVar)
-var C string //@item(cVar, "C", "string", "var")
-
-// //@complete(" ", exportedConst)
-const ExportedConst = "example" //@item(exportedConst, "ExportedConst", "string", "const")
-
-// //@complete(" ", exportedType)
-type ExportedType struct { //@item(exportedType, "ExportedType", "struct{...}", "struct")
-}
-
-// //@complete(" ", exportedFunc)
-func ExportedFunc() int { //@item(exportedFunc, "ExportedFunc", "func() int", "func")
-	return 0
-}
-
-// This tests multiline block comments and completion with prefix
-// Lorem Ipsum Multi//@complete(" ", multilineWithPrefix)
-// Lorem ipsum dolor sit ametom
-func MultilineWithPrefix() int { //@item(multilineWithPrefix, "MultilineWithPrefix", "func() int", "func")
-	return 0
-}
-
 // general completions
 
 type position struct { //@item(structPosition, "position", "struct{...}", "struct")
@@ -32,7 +8,7 @@
 
 func _() {
 	_ = position{
-		//@complete("", fieldX, fieldY, exportedFunc, multilineWithPrefix, structPosition, cVar, exportedConst, exportedType)
+		//@complete("", fieldX, fieldY, structPosition)
 	}
 	_ = position{
 		X: 1,
@@ -44,7 +20,7 @@
 	}
 	_ = []*position{
         {
-            //@complete("", fieldX, fieldY, exportedFunc, multilineWithPrefix, structPosition, cVar, exportedConst, exportedType)
+            //@complete("", fieldX, fieldY, structPosition)
         },
 	}
 }
@@ -60,7 +36,7 @@
 	}
 
 	_ = map[int]int{
-		//@complete("", abVar, exportedFunc, multilineWithPrefix, aaVar, structPosition, cVar, exportedConst, exportedType)
+		//@complete("", abVar, aaVar, structPosition)
 	}
 
 	_ = []string{a: ""} //@complete(":", abVar, aaVar)
@@ -68,7 +44,7 @@
 
 	_ = position{X: a}   //@complete("}", abVar, aaVar)
 	_ = position{a}      //@complete("}", abVar, aaVar)
-	_ = position{a, }      //@complete("}", abVar, exportedFunc, multilineWithPrefix, aaVar, structPosition, cVar, exportedConst, exportedType)
+	_ = position{a, }      //@complete("}", abVar, aaVar, structPosition)
 
 	_ = []int{a}  //@complete("}", abVar, aaVar)
 	_ = [1]int{a} //@complete("}", abVar, aaVar)
@@ -110,7 +86,7 @@
 
 func _() {
 	_ := position{
-		X: 1, //@complete("X", fieldX),complete(" 1", exportedFunc, multilineWithPrefix, structPosition, cVar, exportedConst, exportedType)
-		Y: ,  //@complete(":", fieldY),complete(" ,", exportedFunc, multilineWithPrefix, structPosition, cVar, exportedConst, exportedType)
+		X: 1, //@complete("X", fieldX),complete(" 1", structPosition)
+		Y: ,  //@complete(":", fieldY),complete(" ,", structPosition)
 	}
 }
diff --git a/internal/lsp/testdata/lsp/summary.txt.golden b/internal/lsp/testdata/lsp/summary.txt.golden
index 5d0f432..651e8e9 100644
--- a/internal/lsp/testdata/lsp/summary.txt.golden
+++ b/internal/lsp/testdata/lsp/summary.txt.golden
@@ -1,7 +1,7 @@
 -- summary --
 CallHierarchyCount = 1
 CodeLensCount = 5
-CompletionsCount = 239
+CompletionsCount = 247
 CompletionSnippetCount = 85
 UnimportedCompletionsCount = 6
 DeepCompletionsCount = 5