internal/lsp/semantic: prepare for generics
Semantic tokens will now return typeParameters if the client will accept
them. If the user is not using go1.18, then generic code will not type
check and gopls will not recogize type parameters. If gopls has not
been compiled with go1.18 only some uses of type parameters will be
recognized. In any case the code uses internal/typeparams
to see through IndexListExprs.
There is a global parameter semDebug which should be false in
checked-in code. I use it to see how semantic token decisions are
made; it is for debugging. But there are a lot of small code changes
to restrict logging to a few places.
There is also an unexercised stub to fix a bug where some functions
are misclassified as variables. This will be activated in a future CL,
as it will also change the golden test files.
Change-Id: I5107e67ae25e825b0cdc4c1a744877ce97ba609b
Reviewed-on: https://go-review.googlesource.com/c/tools/+/352570
Trust: Peter Weinberger <pjw@google.com>
Run-TryBot: Peter Weinberger <pjw@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
diff --git a/internal/lsp/semantic.go b/internal/lsp/semantic.go
index 6a98b6c..d59ecad 100644
--- a/internal/lsp/semantic.go
+++ b/internal/lsp/semantic.go
@@ -11,6 +11,7 @@
"go/ast"
"go/token"
"go/types"
+ "log"
"path/filepath"
"sort"
"strings"
@@ -20,6 +21,7 @@
"golang.org/x/tools/internal/lsp/protocol"
"golang.org/x/tools/internal/lsp/source"
"golang.org/x/tools/internal/lsp/template"
+ "golang.org/x/tools/internal/typeparams"
errors "golang.org/x/xerrors"
)
@@ -30,6 +32,10 @@
// reject full semantic token requests for large files
const maxFullFileSize int = 100000
+// to control comprehensive logging of decisions (gopls semtok foo.go > /dev/null shows log output)
+// semDebug should NEVER be true in checked-in code
+const semDebug = false
+
func (s *Server) semanticTokensFull(ctx context.Context, p *protocol.SemanticTokensParams) (*protocol.SemanticTokens, error) {
ret, err := s.computeSemanticTokens(ctx, p.TextDocument, nil)
return ret, err
@@ -153,6 +159,7 @@
tokNamespace tokenType = "namespace"
tokType tokenType = "type"
tokInterface tokenType = "interface"
+ tokTypeParam tokenType = "typeParameter"
tokParameter tokenType = "parameter"
tokVariable tokenType = "variable"
tokMethod tokenType = "method"
@@ -170,7 +177,6 @@
if !start.IsValid() {
// This is not worth reporting
- //e.unexpected("token at token.NoPos")
return
}
if start >= e.end || start+token.Pos(leng) <= e.start {
@@ -233,7 +239,8 @@
// convert the stack to a string, for debugging
func (e *encoded) strStack() string {
msg := []string{"["}
- for _, s := range e.stack {
+ for i := len(e.stack) - 1; i >= 0; i-- {
+ s := e.stack[i]
msg = append(msg, fmt.Sprintf("%T", s)[5:])
}
if len(e.stack) > 0 {
@@ -368,6 +375,7 @@
case *ast.IncDecStmt:
e.token(x.TokPos, len(x.Tok.String()), tokOperator, nil)
case *ast.IndexExpr:
+ case *typeparams.IndexListExpr: // accomodate generics
case *ast.InterfaceType:
e.token(x.Interface, len("interface"), tokKeyword, nil)
case *ast.KeyValueExpr:
@@ -419,7 +427,7 @@
case *ast.Comment, *ast.CommentGroup:
pop()
return false
- default: // just to be super safe.
+ default:
e.unexpected(fmt.Sprintf("failed to implement %T", x))
}
return true
@@ -427,29 +435,55 @@
func (e *encoded) ident(x *ast.Ident) {
if e.ti == nil {
- e.unkIdent(x)
+ what, mods := e.unkIdent(x)
+ if what != "" {
+ e.token(x.Pos(), len(x.String()), what, mods)
+ }
+ if semDebug {
+ log.Printf(" nil %s/nil/nil %q %v %s", x.String(), what, mods, e.strStack())
+ }
return
}
def := e.ti.Defs[x]
if def != nil {
- what, mods := e.definitionFor(x)
+ what, mods := e.definitionFor(x, def)
if what != "" {
e.token(x.Pos(), len(x.String()), what, mods)
}
+ if semDebug {
+ log.Printf(" for %s/%T/%T got %s %v (%s)", x.String(), def, def.Type(), what, mods, e.strStack())
+ }
return
}
use := e.ti.Uses[x]
+ tok := func(pos token.Pos, lng int, tok tokenType, mods []string) {
+ e.token(pos, lng, tok, mods)
+ q := "nil"
+ if use != nil {
+ q = fmt.Sprintf("%T", use.Type())
+ }
+ if semDebug {
+ log.Printf(" use %s/%T/%s got %s %v (%s)", x.String(), use, q, tok, mods, e.strStack())
+ }
+ }
+
switch y := use.(type) {
case nil:
- e.unkIdent(x)
+ what, mods := e.unkIdent(x)
+ if what != "" {
+ tok(x.Pos(), len(x.String()), what, mods)
+ } else if semDebug {
+ // tok() wasn't called, so didn't log
+ log.Printf(" nil %s/%T/nil %q %v (%s)", x.String(), use, what, mods, e.strStack())
+ }
return
case *types.Builtin:
- e.token(x.NamePos, len(x.Name), tokFunction, []string{"defaultLibrary"})
+ tok(x.NamePos, len(x.Name), tokFunction, []string{"defaultLibrary"})
case *types.Const:
mods := []string{"readonly"}
tt := y.Type()
if _, ok := tt.(*types.Basic); ok {
- e.token(x.Pos(), len(x.String()), tokVariable, mods)
+ tok(x.Pos(), len(x.String()), tokVariable, mods)
break
}
if ttx, ok := tt.(*types.Named); ok {
@@ -457,7 +491,7 @@
e.unexpected(fmt.Sprintf("iota:%T", ttx))
}
if _, ok := ttx.Underlying().(*types.Basic); ok {
- e.token(x.Pos(), len(x.String()), tokVariable, mods)
+ tok(x.Pos(), len(x.String()), tokVariable, mods)
break
}
e.unexpected(fmt.Sprintf("%q/%T", x.String(), tt))
@@ -465,22 +499,31 @@
// can this happen? Don't think so
e.unexpected(fmt.Sprintf("%s %T %#v", x.String(), tt, tt))
case *types.Func:
- e.token(x.Pos(), len(x.Name), tokFunction, nil)
+ tok(x.Pos(), len(x.Name), tokFunction, nil)
case *types.Label:
// nothing to map it to
case *types.Nil:
// nil is a predeclared identifier
- e.token(x.Pos(), len("nil"), tokVariable, []string{"readonly", "defaultLibrary"})
+ tok(x.Pos(), len("nil"), tokVariable, []string{"readonly", "defaultLibrary"})
case *types.PkgName:
- e.token(x.Pos(), len(x.Name), tokNamespace, nil)
- case *types.TypeName:
+ tok(x.Pos(), len(x.Name), tokNamespace, nil)
+ case *types.TypeName: // could be a tokTpeParam
var mods []string
if _, ok := y.Type().(*types.Basic); ok {
mods = []string{"defaultLibrary"}
+ } else if _, ok := y.Type().(*typeparams.TypeParam); ok {
+ tok(x.Pos(), len(x.String()), tokTypeParam, mods)
+ break
}
- e.token(x.Pos(), len(x.String()), tokType, mods)
+ tok(x.Pos(), len(x.String()), tokType, mods)
case *types.Var:
- e.token(x.Pos(), len(x.Name), tokVariable, nil)
+ if isSignature(y) {
+ tok(x.Pos(), len(x.Name), tokFunction, nil)
+ } else if _, ok := y.Type().(*typeparams.TypeParam); ok {
+ tok(x.Pos(), len(x.Name), tokTypeParam, nil)
+ } else {
+ tok(x.Pos(), len(x.Name), tokVariable, nil)
+ }
default:
// can't happen
if use == nil {
@@ -495,81 +538,96 @@
}
}
-// both e.ti.Defs and e.ti.Uses are nil. use the parse stack
-// a lot of these only happen when the package doesn't compile
-func (e *encoded) unkIdent(x *ast.Ident) {
- tok := func(tok tokenType, mod []string) {
- e.token(x.Pos(), len(x.Name), tok, mod)
+func isSignature(use types.Object) bool {
+ if true {
+ return false //PJW: fix after generics seem ok
}
+ if _, ok := use.(*types.Var); !ok {
+ return false
+ }
+ v := use.Type()
+ if v == nil {
+ return false
+ }
+ if _, ok := v.(*types.Signature); ok {
+ return true
+ }
+ return false
+}
+
+// both e.ti.Defs and e.ti.Uses are nil. use the parse stack.
+// a lot of these only happen when the package doesn't compile
+// but in that case it is all best-effort from the parse tree
+func (e *encoded) unkIdent(x *ast.Ident) (tokenType, []string) {
def := []string{"definition"}
n := len(e.stack) - 2 // parent of Ident
if n < 0 {
e.unexpected("no stack?")
- return
+ return "", nil
}
switch nd := e.stack[n].(type) {
case *ast.BinaryExpr, *ast.UnaryExpr, *ast.ParenExpr, *ast.StarExpr,
*ast.IncDecStmt, *ast.SliceExpr, *ast.ExprStmt, *ast.IndexExpr,
- *ast.ReturnStmt,
+ *ast.ReturnStmt, *ast.ChanType, *ast.SendStmt,
*ast.ForStmt, // possibly incomplete
*ast.IfStmt, /* condition */
*ast.KeyValueExpr: // either key or value
- tok(tokVariable, nil)
+ return tokVariable, nil
+ case *typeparams.IndexListExpr: // generic?
+ return tokVariable, nil
case *ast.Ellipsis:
- tok(tokType, nil)
+ return tokType, nil
case *ast.CaseClause:
if n-2 >= 0 {
if _, ok := e.stack[n-2].(*ast.TypeSwitchStmt); ok {
- tok(tokType, nil)
- return
+ return tokType, nil
}
}
- tok(tokVariable, nil)
+ return tokVariable, nil
case *ast.ArrayType:
if x == nd.Len {
- tok(tokVariable, nil)
+ // or maybe a Type Param, but we can't just from the parse tree
+ return tokVariable, nil
} else {
- tok(tokType, nil)
+ return tokType, nil
}
case *ast.MapType:
- tok(tokType, nil)
+ return tokType, nil
case *ast.CallExpr:
if x == nd.Fun {
- tok(tokFunction, nil)
- return
+ return tokFunction, nil
}
- tok(tokVariable, nil)
+ return tokVariable, nil
+ case *ast.SwitchStmt:
+ return tokVariable, nil
case *ast.TypeAssertExpr:
if x == nd.X {
- tok(tokVariable, nil)
+ return tokVariable, nil
} else if x == nd.Type {
- tok(tokType, nil)
+ return tokType, nil
}
case *ast.ValueSpec:
for _, p := range nd.Names {
if p == x {
- tok(tokVariable, def)
- return
+ return tokVariable, def
}
}
for _, p := range nd.Values {
if p == x {
- tok(tokVariable, nil)
- return
+ return tokVariable, nil
}
}
- tok(tokType, nil)
+ return tokType, nil
case *ast.SelectorExpr: // e.ti.Selections[nd] is nil, so no help
if n-1 >= 0 {
if ce, ok := e.stack[n-1].(*ast.CallExpr); ok {
// ... CallExpr SelectorExpr Ident (_.x())
if ce.Fun == nd && nd.Sel == x {
- tok(tokFunction, nil)
- return
+ return tokFunction, nil
}
}
}
- tok(tokVariable, nil)
+ return tokVariable, nil
case *ast.AssignStmt:
for _, p := range nd.Lhs {
// x := ..., or x = ...
@@ -577,51 +635,48 @@
if nd.Tok != token.DEFINE {
def = nil
}
- tok(tokVariable, def)
- return
+ return tokVariable, def
}
}
// RHS, = x
- tok(tokVariable, nil)
+ return tokVariable, nil
case *ast.TypeSpec: // it's a type if it is either the Name or the Type
if x == nd.Type {
def = nil
}
- tok(tokType, def)
+ return tokType, def
case *ast.Field:
// ident could be type in a field, or a method in an interface type, or a variable
if x == nd.Type {
- tok(tokType, nil)
- return
+ return tokType, nil
}
if n-2 >= 0 {
_, okit := e.stack[n-2].(*ast.InterfaceType)
_, okfl := e.stack[n-1].(*ast.FieldList)
if okit && okfl {
- tok(tokMethod, def)
- return
+ return tokMethod, def
}
}
- tok(tokVariable, nil)
+ return tokVariable, nil
case *ast.LabeledStmt, *ast.BranchStmt:
// nothing to report
case *ast.CompositeLit:
if nd.Type == x {
- tok(tokType, nil)
- return
+ return tokType, nil
}
- tok(tokVariable, nil)
+ return tokVariable, nil
case *ast.RangeStmt:
if nd.Tok != token.DEFINE {
def = nil
}
- tok(tokVariable, def)
+ return tokVariable, def
case *ast.FuncDecl:
- tok(tokFunction, def)
+ return tokFunction, def
default:
msg := fmt.Sprintf("%T undexpected: %s %s%q", nd, x.Name, e.strStack(), e.srcLine(x))
e.unexpected(msg)
}
+ return "", nil
}
func isDeprecated(n *ast.CommentGroup) bool {
@@ -636,7 +691,9 @@
return false
}
-func (e *encoded) definitionFor(x *ast.Ident) (tokenType, []string) {
+func (e *encoded) definitionFor(x *ast.Ident, def types.Object) (tokenType, []string) {
+ // PJW: def == types.Label? probably a nothing
+ // PJW: look into replaceing these syntactic tests with types more generally
mods := []string{"definition"}
for i := len(e.stack) - 1; i >= 0; i-- {
s := e.stack[i]
@@ -682,6 +739,10 @@
// and in GenDecl/TpeSpec/StructType/FieldList/Field/Ident
// (type A struct{b uint64}
// but on type B struct{C}), C is a type, but is not being defined.
+ // GenDecl/TypeSpec/FieldList/Field/Ident is a typeParam
+ if _, ok := e.stack[i+1].(*ast.FieldList); ok {
+ return tokTypeParam, mods
+ }
fldm := e.stack[len(e.stack)-2]
if fld, ok := fldm.(*ast.Field); ok {
// if len(fld.names) == 0 this is a tokType, being used
@@ -732,7 +793,7 @@
if idx != -1 {
return start + token.Pos(idx)
}
- // can't happen
+ //(in unparsable programs: type _ <-<-chan int)
e.unexpected(fmt.Sprintf("not found:%s %v", keyword, e.fset.PositionFor(start, false)))
return token.NoPos
}
@@ -829,6 +890,9 @@
// log unexpected state
func (e *encoded) unexpected(msg string) {
+ if semDebug {
+ panic(msg)
+ }
event.Error(e.ctx, e.strStack(), errors.New(msg))
}
