gopls/internal/test/integration/misc/semantictokens_test.go - tools - Git at Google

 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package misc

 import (
 	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"

 	"github.com/google/go-cmp/cmp"
 	"golang.org/x/tools/gopls/internal/protocol"
 	. "golang.org/x/tools/gopls/internal/test/integration"
 	"golang.org/x/tools/gopls/internal/test/integration/fake"
 )

 func TestBadURICrash_VSCodeIssue1498(t *testing.T) {
 	const src = `
 -- go.mod --
 module example.com

 go 1.12

 -- main.go --
 package main

 func main() {}

 `
 	WithOptions(
 		Modes(Default),
 	).Run(t, src, func(t *testing.T, env *Env) {
 		params := &protocol.SemanticTokensParams{}
 		const badURI = "http://foo"
 		params.TextDocument.URI = badURI
 		// This call panicked in the past: golang/vscode-go#1498.
 		_, err := env.Editor.Server.SemanticTokensFull(env.Ctx, params)

 		// Requests to an invalid URI scheme now result in an LSP error.
 		got := fmt.Sprint(err)
 		want := `DocumentURI scheme is not 'file': http://foo`
 		if !strings.Contains(got, want) {
 			t.Errorf("SemanticTokensFull error is %v, want substring %q", got, want)
 		}
 	})
 }

 // fix bug involving type parameters and regular parameters
 // (golang/vscode-go#2527)
 func TestSemantic_2527(t *testing.T) {
 	// these are the expected types of identifiers in text order
 	want := []fake.SemanticToken{
 		{Token: "package", TokenType: "keyword"},
 		{Token: "foo", TokenType: "namespace"},
 		{Token: "// comment", TokenType: "comment"},
 		{Token: "func", TokenType: "keyword"},
 		{Token: "Add", TokenType: "function", Mod: "definition signature"},
 		{Token: "T", TokenType: "typeParameter", Mod: "definition"},
 		{Token: "int", TokenType: "type", Mod: "defaultLibrary number"},
 		{Token: "target", TokenType: "parameter", Mod: "definition"},
 		{Token: "T", TokenType: "typeParameter"},
 		{Token: "l", TokenType: "parameter", Mod: "definition slice"},
 		{Token: "T", TokenType: "typeParameter"},
 		{Token: "T", TokenType: "typeParameter"},
 		{Token: "return", TokenType: "keyword"},
 		{Token: "append", TokenType: "function", Mod: "defaultLibrary"},
 		{Token: "l", TokenType: "parameter", Mod: "slice"},
 		{Token: "target", TokenType: "parameter"},
 		{Token: "for", TokenType: "keyword"},
 		{Token: "range", TokenType: "keyword"},
 		{Token: "l", TokenType: "parameter", Mod: "slice"},
 		{Token: "// test coverage", TokenType: "comment"},
 		{Token: "return", TokenType: "keyword"},
 		{Token: "nil", TokenType: "variable", Mod: "readonly defaultLibrary"},
 	}
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo
 // comment
 func Add[T int](target T, l []T) []T {
 	return append(l, target)
 	for range l {} // test coverage
 	return nil
 }
 `
 	WithOptions(
 		Modes(Default),
 		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		env.AfterChange(
 			Diagnostics(env.AtRegexp("main.go", "for range")),
 		)
 		seen := env.SemanticTokensFull("main.go")
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})

 }

 // fix inconsistency in TypeParameters
 // https://github.com/golang/go/issues/57619
 func TestSemantic_57619(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo
 type Smap[K int, V any] struct {
 	Store map[K]V
 }
 func (s *Smap[K, V]) Get(k K) (V, bool) {
 	v, ok := s.Store[k]
 	return v, ok
 }
 func New[K int, V any]() Smap[K, V] {
 	return Smap[K, V]{Store: make(map[K]V)}
 }
 `
 	WithOptions(
 		Modes(Default),
 		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
 		for i, s := range seen {
 			if (s.Token == "K" || s.Token == "V") && s.TokenType != "typeParameter" {
 				t.Errorf("%d: expected K and V to be type parameters, but got %v", i, s)
 			}
 		}
 	})
 }

 func TestSemanticGoDirectives(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo

 //go:linkname now time.Now
 func now()

 //go:noinline
 func foo() {}

 // Mentioning go:noinline should not tokenize.

 //go:notadirective
 func bar() {}
 `
 	want := []fake.SemanticToken{
 		{Token: "package", TokenType: "keyword"},
 		{Token: "foo", TokenType: "namespace"},

 		{Token: "//", TokenType: "comment"},
 		{Token: "go:linkname", TokenType: "namespace"},
 		{Token: "now time.Now", TokenType: "comment"},
 		{Token: "func", TokenType: "keyword"},
 		{Token: "now", TokenType: "function", Mod: "definition signature"},

 		{Token: "//", TokenType: "comment"},
 		{Token: "go:noinline", TokenType: "namespace"},
 		{Token: "func", TokenType: "keyword"},
 		{Token: "foo", TokenType: "function", Mod: "definition signature"},

 		{Token: "// Mentioning go:noinline should not tokenize.", TokenType: "comment"},

 		{Token: "//go:notadirective", TokenType: "comment"},
 		{Token: "func", TokenType: "keyword"},
 		{Token: "bar", TokenType: "function", Mod: "definition signature"},
 	}

 	WithOptions(
 		Modes(Default),
 		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }

 // Make sure no zero-length tokens occur
 func TestSemantic_65254(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.21
 -- main.go --
 package main

 /* a comment with an

 empty line
 */

 const bad = `

 	src += "`foo" + `
 	` + "bar`"
 	want := []fake.SemanticToken{
 		{Token: "package", TokenType: "keyword"},
 		{Token: "main", TokenType: "namespace"},
 		{Token: "/* a comment with an", TokenType: "comment"},
 		// --- Note that the zero length line does not show up
 		{Token: "empty line", TokenType: "comment"},
 		{Token: "*/", TokenType: "comment"},
 		{Token: "const", TokenType: "keyword"},
 		{Token: "bad", TokenType: "variable", Mod: "definition readonly"},
 		{Token: "`foo", TokenType: "string"},
 		// --- Note the zero length line does not show up
 		{Token: "\tbar`", TokenType: "string"},
 	}
 	WithOptions(
 		Modes(Default),
 		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }

 // Check delta-encoding of token line is sound even when the first
 // token is discarded (go.dev/issue/78269).
 func TestSemanticTokens_firstSuppressed(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com
 go 1.21

 -- p.go --
 // comment
 package p

 // imports
 import _ "fmt"

 func F()
 `
 	WithOptions(
 		Modes(Default),
 		Settings{
 			"semanticTokens": true,
 			"semanticTokenTypes": map[string]any{
 				"keyword":   false,
 				"namespace": false,
 				"comment":   false,
 			},
 		},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("p.go")
 		seen := env.SemanticTokensFull("p.go")
 		want := []fake.SemanticToken{
 			// These tokens are suppressed by the settings above:
 			//  {Token: "// comment", TokenType: "comment"},
 			//  {Token: "package", TokenType: "keyword"},
 			//  {Token: "p", TokenType: "namespace"},
 			//  {Token: "// imports", TokenType: "comment"},
 			//  {Token: "import", TokenType: "keyword"},
 			//  {Token: "func", TokenType: "keyword"},
 			// This first token must be reported at "F":
 			{Token: "F", TokenType: "function", Mod: "definition signature"},
 		}
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }

 // This test exercises the UTF-16 logic in Editor.interpretTokens.
 func TestSemanticTokens_multiline(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com
 go 1.21

 -- p.go --
 package p
 ` + "const _ = `世\n😃`"

 	WithOptions(
 		Modes(Default),
 		Settings{
 			"semanticTokens": true,
 			"semanticTokenTypes": map[string]any{
 				"package":   false,
 				"keyword":   false,
 				"namespace": false,
 				"variable":  false,
 				"string":    true,
 			},
 		},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("p.go")
 		seen := env.SemanticTokensFull("p.go")
 		want := []fake.SemanticToken{
 			{Token: "`世", TokenType: "string"},
 			{Token: "😃`", TokenType: "string"},
 		}
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }

 // This sets client capability fields:
 // - `textDocument.semanticTokens.tokenTypes`
 // - `textDocument.semanticTokens.tokenModifiers`
 func tokenClientCapabilities(t *testing.T, tokenTypes, tokenMods []string) RunOption {
 	t.Helper()
 	typeString, err := json.Marshal(tokenTypes)
 	if err != nil {
 		t.Fatalf("accepted token list failed to marshal: %v", err)
 	}
 	modString, err := json.Marshal(tokenMods)
 	if err != nil {
 		t.Fatalf("accepted mod list failed to marshal: %v", err)
 	}
 	// This will only update the specific client capabilities set.
 	return CapabilitiesJSON(fmt.Appendf(nil,
 		`{"textDocument": {"semanticTokens": {"tokenTypes": %s, "tokenModifiers": %s}}}`,
 		typeString, modString))
 }

 // This tests the Client Capabilities:
 // -  `textDocument.semanticTokens.tokenTypes`
 // -  `textDocument.semanticTokens.tokenModifiers`
 //
 // These two client capabilities determine the maximum set of types and mods
 // the server can send back to the client. So the intersection of these lists
 // with the lists of what the server supports is what gopls can return.
 func TestSemanticTokenClientCapabilitiesTypeAndModifier(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo

 // Comment
 type A struct {}
 `
 	tcs := []struct {
 		name           string
 		tokenTypes     []string
 		tokenModifiers []string
 		want           []fake.SemanticToken
 	}{{
 		name:           "base case",
 		tokenTypes:     []string{"comment", "keyword", "namespace", "type"},
 		tokenModifiers: []string{"definition", "struct"},
 		want: []fake.SemanticToken{
 			{Token: "package", TokenType: "keyword"},
 			{Token: "foo", TokenType: "namespace"},

 			{Token: "// Comment", TokenType: "comment"},
 			{Token: "type", TokenType: "keyword"},
 			{Token: "A", TokenType: "type", Mod: "definition struct"},
 			{Token: "struct", TokenType: "keyword"},
 		},
 	}, {
 		name:           "support subset of token types",
 		tokenTypes:     []string{"comment", "type"},
 		tokenModifiers: []string{"definition", "struct"},
 		want: []fake.SemanticToken{
 			{Token: "// Comment", TokenType: "comment"},
 			{Token: "A", TokenType: "type", Mod: "definition struct"},
 		},
 	}, {
 		name:           "support subset of modifiers",
 		tokenTypes:     []string{"comment", "keyword", "namespace", "type"},
 		tokenModifiers: []string{"definition"},
 		want: []fake.SemanticToken{
 			{Token: "package", TokenType: "keyword"},
 			{Token: "foo", TokenType: "namespace"},

 			{Token: "// Comment", TokenType: "comment"},
 			{Token: "type", TokenType: "keyword"},
 			// "struct" modifier no longer supported
 			{Token: "A", TokenType: "type", Mod: "definition"},
 			{Token: "struct", TokenType: "keyword"},
 		},
 	},
 	}

 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
 			WithOptions(
 				Modes(Default),
 				tokenClientCapabilities(t, tc.tokenTypes, tc.tokenModifiers),
 				Settings{"semanticTokens": true},
 			).Run(t, src, func(t *testing.T, env *Env) {
 				env.OpenFile("main.go")
 				seen := env.SemanticTokensFull("main.go")
 				if x := cmp.Diff(tc.want, seen); x != "" {
 					t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 				}
 			})
 		})
 	}
 }

 // This test verifies that UI Setting's `semanticTokens` map only further
 // restricts the client capabilities `tokenTypes` list.
 //
 // TODO(aputman): Consider moving to marker test.
 func TestSemanticTokenTypesClientCapabilitiesWithUISettings(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo

 // Comment
 type A struct {}
 `
 	// What the client capability token types are.
 	clientTokenTypes := []string{"comment", "keyword", "type"}

 	// The UI Settings token types map
 	uiTokenTypes := map[string]bool{
 		// Since "namespace" isn't in the clientTokenTypes list, this
 		// won't do anything.
 		"namespace": true,
 		"type":      true,
 		// This will disable the keyword tokentypes even though the client
 		// supports them.
 		"keyword": false,
 		// Since "comment" is left out, it will be supported.
 	}

 	clientTokenModifiers := []string{"definition", "struct"}

 	// Only comment and type are left
 	want := []fake.SemanticToken{
 		{Token: "// Comment", TokenType: "comment"},
 		{Token: "A", TokenType: "type", Mod: "definition struct"},
 	}
 	WithOptions(
 		Modes(Default),
 		tokenClientCapabilities(t, clientTokenTypes, clientTokenModifiers),
 		Settings{"semanticTokens": true, "semanticTokenTypes": uiTokenTypes},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }

 // This test verifies that UI Setting's semanticTokenModifiers map only
 // further restricts the client capabilities tokenModifiers list.
 //
 // TODO(aputman): Consider moving to marker test.
 func TestSemanticTokenModifiersClientCapabilitiesWithUISettings(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com

 go 1.19
 -- main.go --
 package foo

 type A struct {}

 func B(l []string) {}
 `
 	// Only these token types are returned.
 	clientTokenTypes := []string{"comment", "definition", "function", "parameter", "type"}

 	// What the client capability token modifiers are.
 	clientTokenModifiers := []string{"definition", "signature", "struct"}

 	// The UI Settings token mods map
 	uiTokenMods := map[string]bool{
 		// Since "slice" isn't in the clientTokenModifiers list, this won't do anything.
 		"slice":     true,
 		"signature": true,
 		// This will disable the definition modifiers even though the client supports them.
 		"definition": false,
 		// Since "struct" is left out, it will be supported.
 	}

 	// Only signature and struct modifiers are left
 	want := []fake.SemanticToken{
 		{Token: "A", TokenType: "type", Mod: "struct"},        // "definition" no longer returned
 		{Token: "B", TokenType: "function", Mod: "signature"}, // "definition" no longer returned
 		{Token: "l", TokenType: "parameter", Mod: ""},         // "slice" no longer returned
 		{Token: "string", TokenType: "type", Mod: ""},
 	}

 	WithOptions(
 		Modes(Default),
 		tokenClientCapabilities(t, clientTokenTypes, clientTokenModifiers),
 		Settings{"semanticTokens": true, "semanticTokenModifiers": uiTokenMods},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
 		if x := cmp.Diff(want, seen); x != "" {
 			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
 		}
 	})
 }
	// Copyright 2021 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package misc

	import (
	"encoding/json"
	"fmt"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	"golang.org/x/tools/gopls/internal/protocol"
	. "golang.org/x/tools/gopls/internal/test/integration"
	"golang.org/x/tools/gopls/internal/test/integration/fake"
	)

	func TestBadURICrash_VSCodeIssue1498(t *testing.T) {
	const src = `
	-- go.mod --
	module example.com

	go 1.12

	-- main.go --
	package main

	func main() {}

	`
	WithOptions(
	Modes(Default),
	).Run(t, src, func(t testing.T, env Env) {
	params := &protocol.SemanticTokensParams{}
	const badURI = "http://foo"
	params.TextDocument.URI = badURI
	// This call panicked in the past: golang/vscode-go#1498.
	_, err := env.Editor.Server.SemanticTokensFull(env.Ctx, params)

	// Requests to an invalid URI scheme now result in an LSP error.
	got := fmt.Sprint(err)
	want := `DocumentURI scheme is not 'file': http://foo`
	if !strings.Contains(got, want) {
	t.Errorf("SemanticTokensFull error is %v, want substring %q", got, want)
	}
	})
	}

	// fix bug involving type parameters and regular parameters
	// (golang/vscode-go#2527)
	func TestSemantic_2527(t *testing.T) {
	// these are the expected types of identifiers in text order
	want := []fake.SemanticToken{
	{Token: "package", TokenType: "keyword"},
	{Token: "foo", TokenType: "namespace"},
	{Token: "// comment", TokenType: "comment"},
	{Token: "func", TokenType: "keyword"},
	{Token: "Add", TokenType: "function", Mod: "definition signature"},
	{Token: "T", TokenType: "typeParameter", Mod: "definition"},
	{Token: "int", TokenType: "type", Mod: "defaultLibrary number"},
	{Token: "target", TokenType: "parameter", Mod: "definition"},
	{Token: "T", TokenType: "typeParameter"},
	{Token: "l", TokenType: "parameter", Mod: "definition slice"},
	{Token: "T", TokenType: "typeParameter"},
	{Token: "T", TokenType: "typeParameter"},
	{Token: "return", TokenType: "keyword"},
	{Token: "append", TokenType: "function", Mod: "defaultLibrary"},
	{Token: "l", TokenType: "parameter", Mod: "slice"},
	{Token: "target", TokenType: "parameter"},
	{Token: "for", TokenType: "keyword"},
	{Token: "range", TokenType: "keyword"},
	{Token: "l", TokenType: "parameter", Mod: "slice"},
	{Token: "// test coverage", TokenType: "comment"},
	{Token: "return", TokenType: "keyword"},
	{Token: "nil", TokenType: "variable", Mod: "readonly defaultLibrary"},
	}
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo
	// comment
	func Add[T int](target T, l []T) []T {
	return append(l, target)
	for range l {} // test coverage
	return nil
	}
	`
	WithOptions(
	Modes(Default),
	Settings{"semanticTokens": true},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	env.AfterChange(
	Diagnostics(env.AtRegexp("main.go", "for range")),
	)
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})

	}

	// fix inconsistency in TypeParameters
	// https://github.com/golang/go/issues/57619
	func TestSemantic_57619(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo
	type Smap[K int, V any] struct {
	Store map[K]V
	}
	func (s *Smap[K, V]) Get(k K) (V, bool) {
	v, ok := s.Store[k]
	return v, ok
	}
	func New[K int, V any]() Smap[K, V] {
	return Smap[K, V]{Store: make(map[K]V)}
	}
	`
	WithOptions(
	Modes(Default),
	Settings{"semanticTokens": true},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	for i, s := range seen {
	if (s.Token == "K" \|\| s.Token == "V") && s.TokenType != "typeParameter" {
	t.Errorf("%d: expected K and V to be type parameters, but got %v", i, s)
	}
	}
	})
	}

	func TestSemanticGoDirectives(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo

	//go:linkname now time.Now
	func now()

	//go:noinline
	func foo() {}

	// Mentioning go:noinline should not tokenize.

	//go:notadirective
	func bar() {}
	`
	want := []fake.SemanticToken{
	{Token: "package", TokenType: "keyword"},
	{Token: "foo", TokenType: "namespace"},

	{Token: "//", TokenType: "comment"},
	{Token: "go:linkname", TokenType: "namespace"},
	{Token: "now time.Now", TokenType: "comment"},
	{Token: "func", TokenType: "keyword"},
	{Token: "now", TokenType: "function", Mod: "definition signature"},

	{Token: "//", TokenType: "comment"},
	{Token: "go:noinline", TokenType: "namespace"},
	{Token: "func", TokenType: "keyword"},
	{Token: "foo", TokenType: "function", Mod: "definition signature"},

	{Token: "// Mentioning go:noinline should not tokenize.", TokenType: "comment"},

	{Token: "//go:notadirective", TokenType: "comment"},
	{Token: "func", TokenType: "keyword"},
	{Token: "bar", TokenType: "function", Mod: "definition signature"},
	}

	WithOptions(
	Modes(Default),
	Settings{"semanticTokens": true},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}

	// Make sure no zero-length tokens occur
	func TestSemantic_65254(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.21
	-- main.go --
	package main

	/* a comment with an

	empty line
	*/

	const bad = `

	src += "`foo" + `
	` + "bar`"
	want := []fake.SemanticToken{
	{Token: "package", TokenType: "keyword"},
	{Token: "main", TokenType: "namespace"},
	{Token: "/* a comment with an", TokenType: "comment"},
	// --- Note that the zero length line does not show up
	{Token: "empty line", TokenType: "comment"},
	{Token: "*/", TokenType: "comment"},
	{Token: "const", TokenType: "keyword"},
	{Token: "bad", TokenType: "variable", Mod: "definition readonly"},
	{Token: "`foo", TokenType: "string"},
	// --- Note the zero length line does not show up
	{Token: "\tbar`", TokenType: "string"},
	}
	WithOptions(
	Modes(Default),
	Settings{"semanticTokens": true},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}

	// Check delta-encoding of token line is sound even when the first
	// token is discarded (go.dev/issue/78269).
	func TestSemanticTokens_firstSuppressed(t *testing.T) {
	src := `
	-- go.mod --
	module example.com
	go 1.21

	-- p.go --
	// comment
	package p

	// imports
	import _ "fmt"

	func F()
	`
	WithOptions(
	Modes(Default),
	Settings{
	"semanticTokens": true,
	"semanticTokenTypes": map[string]any{
	"keyword": false,
	"namespace": false,
	"comment": false,
	},
	},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("p.go")
	seen := env.SemanticTokensFull("p.go")
	want := []fake.SemanticToken{
	// These tokens are suppressed by the settings above:
	// {Token: "// comment", TokenType: "comment"},
	// {Token: "package", TokenType: "keyword"},
	// {Token: "p", TokenType: "namespace"},
	// {Token: "// imports", TokenType: "comment"},
	// {Token: "import", TokenType: "keyword"},
	// {Token: "func", TokenType: "keyword"},
	// This first token must be reported at "F":
	{Token: "F", TokenType: "function", Mod: "definition signature"},
	}
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}

	// This test exercises the UTF-16 logic in Editor.interpretTokens.
	func TestSemanticTokens_multiline(t *testing.T) {
	src := `
	-- go.mod --
	module example.com
	go 1.21

	-- p.go --
	package p
	` + "const _ = `世\n😃`"

	WithOptions(
	Modes(Default),
	Settings{
	"semanticTokens": true,
	"semanticTokenTypes": map[string]any{
	"package": false,
	"keyword": false,
	"namespace": false,
	"variable": false,
	"string": true,
	},
	},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("p.go")
	seen := env.SemanticTokensFull("p.go")
	want := []fake.SemanticToken{
	{Token: "`世", TokenType: "string"},
	{Token: "😃`", TokenType: "string"},
	}
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}

	// This sets client capability fields:
	// - `textDocument.semanticTokens.tokenTypes`
	// - `textDocument.semanticTokens.tokenModifiers`
	func tokenClientCapabilities(t *testing.T, tokenTypes, tokenMods []string) RunOption {
	t.Helper()
	typeString, err := json.Marshal(tokenTypes)
	if err != nil {
	t.Fatalf("accepted token list failed to marshal: %v", err)
	}
	modString, err := json.Marshal(tokenMods)
	if err != nil {
	t.Fatalf("accepted mod list failed to marshal: %v", err)
	}
	// This will only update the specific client capabilities set.
	return CapabilitiesJSON(fmt.Appendf(nil,
	`{"textDocument": {"semanticTokens": {"tokenTypes": %s, "tokenModifiers": %s}}}`,
	typeString, modString))
	}

	// This tests the Client Capabilities:
	// - `textDocument.semanticTokens.tokenTypes`
	// - `textDocument.semanticTokens.tokenModifiers`
	//
	// These two client capabilities determine the maximum set of types and mods
	// the server can send back to the client. So the intersection of these lists
	// with the lists of what the server supports is what gopls can return.
	func TestSemanticTokenClientCapabilitiesTypeAndModifier(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo

	// Comment
	type A struct {}
	`
	tcs := []struct {
	name string
	tokenTypes []string
	tokenModifiers []string
	want []fake.SemanticToken
	}{{
	name: "base case",
	tokenTypes: []string{"comment", "keyword", "namespace", "type"},
	tokenModifiers: []string{"definition", "struct"},
	want: []fake.SemanticToken{
	{Token: "package", TokenType: "keyword"},
	{Token: "foo", TokenType: "namespace"},

	{Token: "// Comment", TokenType: "comment"},
	{Token: "type", TokenType: "keyword"},
	{Token: "A", TokenType: "type", Mod: "definition struct"},
	{Token: "struct", TokenType: "keyword"},
	},
	}, {
	name: "support subset of token types",
	tokenTypes: []string{"comment", "type"},
	tokenModifiers: []string{"definition", "struct"},
	want: []fake.SemanticToken{
	{Token: "// Comment", TokenType: "comment"},
	{Token: "A", TokenType: "type", Mod: "definition struct"},
	},
	}, {
	name: "support subset of modifiers",
	tokenTypes: []string{"comment", "keyword", "namespace", "type"},
	tokenModifiers: []string{"definition"},
	want: []fake.SemanticToken{
	{Token: "package", TokenType: "keyword"},
	{Token: "foo", TokenType: "namespace"},

	{Token: "// Comment", TokenType: "comment"},
	{Token: "type", TokenType: "keyword"},
	// "struct" modifier no longer supported
	{Token: "A", TokenType: "type", Mod: "definition"},
	{Token: "struct", TokenType: "keyword"},
	},
	},
	}

	for _, tc := range tcs {
	t.Run(tc.name, func(t *testing.T) {
	WithOptions(
	Modes(Default),
	tokenClientCapabilities(t, tc.tokenTypes, tc.tokenModifiers),
	Settings{"semanticTokens": true},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(tc.want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	})
	}
	}

	// This test verifies that UI Setting's `semanticTokens` map only further
	// restricts the client capabilities `tokenTypes` list.
	//
	// TODO(aputman): Consider moving to marker test.
	func TestSemanticTokenTypesClientCapabilitiesWithUISettings(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo

	// Comment
	type A struct {}
	`
	// What the client capability token types are.
	clientTokenTypes := []string{"comment", "keyword", "type"}

	// The UI Settings token types map
	uiTokenTypes := map[string]bool{
	// Since "namespace" isn't in the clientTokenTypes list, this
	// won't do anything.
	"namespace": true,
	"type": true,
	// This will disable the keyword tokentypes even though the client
	// supports them.
	"keyword": false,
	// Since "comment" is left out, it will be supported.
	}

	clientTokenModifiers := []string{"definition", "struct"}

	// Only comment and type are left
	want := []fake.SemanticToken{
	{Token: "// Comment", TokenType: "comment"},
	{Token: "A", TokenType: "type", Mod: "definition struct"},
	}
	WithOptions(
	Modes(Default),
	tokenClientCapabilities(t, clientTokenTypes, clientTokenModifiers),
	Settings{"semanticTokens": true, "semanticTokenTypes": uiTokenTypes},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}

	// This test verifies that UI Setting's semanticTokenModifiers map only
	// further restricts the client capabilities tokenModifiers list.
	//
	// TODO(aputman): Consider moving to marker test.
	func TestSemanticTokenModifiersClientCapabilitiesWithUISettings(t *testing.T) {
	src := `
	-- go.mod --
	module example.com

	go 1.19
	-- main.go --
	package foo

	type A struct {}

	func B(l []string) {}
	`
	// Only these token types are returned.
	clientTokenTypes := []string{"comment", "definition", "function", "parameter", "type"}

	// What the client capability token modifiers are.
	clientTokenModifiers := []string{"definition", "signature", "struct"}

	// The UI Settings token mods map
	uiTokenMods := map[string]bool{
	// Since "slice" isn't in the clientTokenModifiers list, this won't do anything.
	"slice": true,
	"signature": true,
	// This will disable the definition modifiers even though the client supports them.
	"definition": false,
	// Since "struct" is left out, it will be supported.
	}

	// Only signature and struct modifiers are left
	want := []fake.SemanticToken{
	{Token: "A", TokenType: "type", Mod: "struct"}, // "definition" no longer returned
	{Token: "B", TokenType: "function", Mod: "signature"}, // "definition" no longer returned
	{Token: "l", TokenType: "parameter", Mod: ""}, // "slice" no longer returned
	{Token: "string", TokenType: "type", Mod: ""},
	}

	WithOptions(
	Modes(Default),
	tokenClientCapabilities(t, clientTokenTypes, clientTokenModifiers),
	Settings{"semanticTokens": true, "semanticTokenModifiers": uiTokenMods},
	).Run(t, src, func(t testing.T, env Env) {
	env.OpenFile("main.go")
	seen := env.SemanticTokensFull("main.go")
	if x := cmp.Diff(want, seen); x != "" {
	t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
	}
	})
	}