[release-branch.0.24] internal/tokeninternal: move unsafe gopls code into gopls
In x/tools@v0.24.0, the tokeninternal package accessed the token.FileSet
unsafely, and used a trick to force the build to break if ever the size
of token.FileSet changed. This workaround was only necessary for gopls,
but was unfortunately implemented in a package that was reachable
through the public API of x/tools.
In go1.25, the size of the token.FileSet changed, breaking tools that
indirectly depended on this poisoned package. Fix this by moving the
gopls logic into gopls: the gopls build will still be broken, but that
doesn't matter as we only support the most recent version of gopls,
which is fixed.
For golang/go#74462
Change-Id: Ied3f7a615cab66d88ec6f4cfe8738f8429865993
Reviewed-on: https://go-review.googlesource.com/c/tools/+/697336
Reviewed-by: Peter Weinberger <pjw@google.com>
TryBot-Bypass: Robert Findley <rfindley@google.com>
diff --git a/gopls/internal/cache/check.go b/gopls/internal/cache/check.go
index ea91d28..57e2c1f 100644
--- a/gopls/internal/cache/check.go
+++ b/gopls/internal/cache/check.go
@@ -30,6 +30,7 @@
"golang.org/x/tools/gopls/internal/filecache"
"golang.org/x/tools/gopls/internal/label"
"golang.org/x/tools/gopls/internal/protocol"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/util/safetoken"
"golang.org/x/tools/gopls/internal/util/slices"
@@ -37,7 +38,6 @@
"golang.org/x/tools/internal/event"
"golang.org/x/tools/internal/gcimporter"
"golang.org/x/tools/internal/packagesinternal"
- "golang.org/x/tools/internal/tokeninternal"
"golang.org/x/tools/internal/typesinternal"
"golang.org/x/tools/internal/versions"
)
diff --git a/gopls/internal/cache/parse_cache.go b/gopls/internal/cache/parse_cache.go
index 8586f65..2302fa0 100644
--- a/gopls/internal/cache/parse_cache.go
+++ b/gopls/internal/cache/parse_cache.go
@@ -20,8 +20,8 @@
"golang.org/x/tools/gopls/internal/cache/parsego"
"golang.org/x/tools/gopls/internal/file"
"golang.org/x/tools/gopls/internal/protocol"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/internal/memoize"
- "golang.org/x/tools/internal/tokeninternal"
)
// This file contains an implementation of an LRU parse cache, that offsets the
diff --git a/gopls/internal/cache/parsego/parse_test.go b/gopls/internal/cache/parsego/parse_test.go
index c641254..3799374 100644
--- a/gopls/internal/cache/parsego/parse_test.go
+++ b/gopls/internal/cache/parsego/parse_test.go
@@ -11,8 +11,8 @@
"testing"
"golang.org/x/tools/gopls/internal/cache/parsego"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/safetoken"
- "golang.org/x/tools/internal/tokeninternal"
)
// TODO(golang/go#64335): we should have many more tests for fixed syntax.
diff --git a/gopls/internal/golang/change_signature.go b/gopls/internal/golang/change_signature.go
index 72cbe4c..699e0b1 100644
--- a/gopls/internal/golang/change_signature.go
+++ b/gopls/internal/golang/change_signature.go
@@ -20,13 +20,13 @@
"golang.org/x/tools/gopls/internal/cache/parsego"
"golang.org/x/tools/gopls/internal/file"
"golang.org/x/tools/gopls/internal/protocol"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/util/safetoken"
"golang.org/x/tools/imports"
internalastutil "golang.org/x/tools/internal/astutil"
"golang.org/x/tools/internal/diff"
"golang.org/x/tools/internal/refactor/inline"
- "golang.org/x/tools/internal/tokeninternal"
"golang.org/x/tools/internal/typesinternal"
"golang.org/x/tools/internal/versions"
)
diff --git a/gopls/internal/golang/format.go b/gopls/internal/golang/format.go
index 5755f7a..0de1bc8 100644
--- a/gopls/internal/golang/format.go
+++ b/gopls/internal/golang/format.go
@@ -22,11 +22,11 @@
"golang.org/x/tools/gopls/internal/file"
"golang.org/x/tools/gopls/internal/protocol"
"golang.org/x/tools/gopls/internal/settings"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/safetoken"
"golang.org/x/tools/internal/diff"
"golang.org/x/tools/internal/event"
"golang.org/x/tools/internal/imports"
- "golang.org/x/tools/internal/tokeninternal"
)
// Format formats a file with a given range.
diff --git a/gopls/internal/golang/hover.go b/gopls/internal/golang/hover.go
index 2edb8a9..2fda396 100644
--- a/gopls/internal/golang/hover.go
+++ b/gopls/internal/golang/hover.go
@@ -33,6 +33,7 @@
"golang.org/x/tools/gopls/internal/file"
"golang.org/x/tools/gopls/internal/protocol"
"golang.org/x/tools/gopls/internal/settings"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
gastutil "golang.org/x/tools/gopls/internal/util/astutil"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/util/safetoken"
@@ -41,7 +42,6 @@
"golang.org/x/tools/internal/aliases"
"golang.org/x/tools/internal/event"
"golang.org/x/tools/internal/stdlib"
- "golang.org/x/tools/internal/tokeninternal"
"golang.org/x/tools/internal/typeparams"
"golang.org/x/tools/internal/typesinternal"
)
diff --git a/gopls/internal/golang/stub.go b/gopls/internal/golang/stub.go
index 47bcf3a..afc448d 100644
--- a/gopls/internal/golang/stub.go
+++ b/gopls/internal/golang/stub.go
@@ -22,10 +22,10 @@
"golang.org/x/tools/gopls/internal/cache"
"golang.org/x/tools/gopls/internal/cache/metadata"
"golang.org/x/tools/gopls/internal/cache/parsego"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/util/safetoken"
"golang.org/x/tools/internal/diff"
- "golang.org/x/tools/internal/tokeninternal"
)
// stubMethodsFixer returns a suggested fix to declare the missing
diff --git a/gopls/internal/golang/types_format.go b/gopls/internal/golang/types_format.go
index 51584bc..13730a0 100644
--- a/gopls/internal/golang/types_format.go
+++ b/gopls/internal/golang/types_format.go
@@ -18,9 +18,9 @@
"golang.org/x/tools/gopls/internal/cache"
"golang.org/x/tools/gopls/internal/protocol"
"golang.org/x/tools/gopls/internal/settings"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/internal/event"
- "golang.org/x/tools/internal/tokeninternal"
"golang.org/x/tools/internal/typeparams"
)
diff --git a/gopls/internal/golang/util.go b/gopls/internal/golang/util.go
index 18f7242..ec2e6a7 100644
--- a/gopls/internal/golang/util.go
+++ b/gopls/internal/golang/util.go
@@ -17,10 +17,10 @@
"golang.org/x/tools/gopls/internal/cache/metadata"
"golang.org/x/tools/gopls/internal/cache/parsego"
"golang.org/x/tools/gopls/internal/protocol"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/astutil"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/util/safetoken"
- "golang.org/x/tools/internal/tokeninternal"
)
// IsGenerated gets and reads the file denoted by uri and reports
diff --git a/gopls/internal/server/command.go b/gopls/internal/server/command.go
index 25a7f33..c667df6 100644
--- a/gopls/internal/server/command.go
+++ b/gopls/internal/server/command.go
@@ -34,13 +34,13 @@
"golang.org/x/tools/gopls/internal/protocol"
"golang.org/x/tools/gopls/internal/protocol/command"
"golang.org/x/tools/gopls/internal/settings"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
"golang.org/x/tools/gopls/internal/util/bug"
"golang.org/x/tools/gopls/internal/vulncheck"
"golang.org/x/tools/gopls/internal/vulncheck/scan"
"golang.org/x/tools/internal/diff"
"golang.org/x/tools/internal/event"
"golang.org/x/tools/internal/gocommand"
- "golang.org/x/tools/internal/tokeninternal"
"golang.org/x/tools/internal/xcontext"
)
diff --git a/gopls/internal/tokeninternal/tokeninternal.go b/gopls/internal/tokeninternal/tokeninternal.go
new file mode 100644
index 0000000..a0b6c7f
--- /dev/null
+++ b/gopls/internal/tokeninternal/tokeninternal.go
@@ -0,0 +1,107 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tokeninternal was split off from x/tools/internal/tokeninternal, to
+// avoid the build breakage described in golang/go#74462.
+package tokeninternal
+
+import (
+ "fmt"
+ "go/token"
+ "sort"
+ "sync"
+ "unsafe"
+
+ "golang.org/x/tools/internal/tokeninternal"
+)
+
+// AddExistingFiles adds the specified files to the FileSet if they
+// are not already present. It panics if any pair of files in the
+// resulting FileSet would overlap.
+func AddExistingFiles(fset *token.FileSet, files []*token.File) {
+ // Punch through the FileSet encapsulation.
+ type tokenFileSet struct {
+ // This type remained essentially consistent from go1.16 to go1.21.
+ mutex sync.RWMutex
+ base int
+ files []*token.File
+ _ *token.File // changed to atomic.Pointer[token.File] in go1.19
+ }
+
+ // If the size of token.FileSet changes, this will fail to compile.
+ const delta = int64(unsafe.Sizeof(tokenFileSet{})) - int64(unsafe.Sizeof(token.FileSet{}))
+ var _ [-delta * delta]int
+
+ type uP = unsafe.Pointer
+ var ptr *tokenFileSet
+ *(*uP)(uP(&ptr)) = uP(fset)
+ ptr.mutex.Lock()
+ defer ptr.mutex.Unlock()
+
+ // Merge and sort.
+ newFiles := append(ptr.files, files...)
+ sort.Slice(newFiles, func(i, j int) bool {
+ return newFiles[i].Base() < newFiles[j].Base()
+ })
+
+ // Reject overlapping files.
+ // Discard adjacent identical files.
+ out := newFiles[:0]
+ for i, file := range newFiles {
+ if i > 0 {
+ prev := newFiles[i-1]
+ if file == prev {
+ continue
+ }
+ if prev.Base()+prev.Size()+1 > file.Base() {
+ panic(fmt.Sprintf("file %s (%d-%d) overlaps with file %s (%d-%d)",
+ prev.Name(), prev.Base(), prev.Base()+prev.Size(),
+ file.Name(), file.Base(), file.Base()+file.Size()))
+ }
+ }
+ out = append(out, file)
+ }
+ newFiles = out
+
+ ptr.files = newFiles
+
+ // Advance FileSet.Base().
+ if len(newFiles) > 0 {
+ last := newFiles[len(newFiles)-1]
+ newBase := last.Base() + last.Size() + 1
+ if ptr.base < newBase {
+ ptr.base = newBase
+ }
+ }
+}
+
+// FileSetFor returns a new FileSet containing a sequence of new Files with
+// the same base, size, and line as the input files, for use in APIs that
+// require a FileSet.
+//
+// Precondition: the input files must be non-overlapping, and sorted in order
+// of their Base.
+func FileSetFor(files ...*token.File) *token.FileSet {
+ fset := token.NewFileSet()
+ for _, f := range files {
+ f2 := fset.AddFile(f.Name(), f.Base(), f.Size())
+ lines := tokeninternal.GetLines(f)
+ f2.SetLines(lines)
+ }
+ return fset
+}
+
+// CloneFileSet creates a new FileSet holding all files in fset. It does not
+// create copies of the token.Files in fset: they are added to the resulting
+// FileSet unmodified.
+func CloneFileSet(fset *token.FileSet) *token.FileSet {
+ var files []*token.File
+ fset.Iterate(func(f *token.File) bool {
+ files = append(files, f)
+ return true
+ })
+ newFileSet := token.NewFileSet()
+ AddExistingFiles(newFileSet, files)
+ return newFileSet
+}
diff --git a/internal/tokeninternal/tokeninternal_test.go b/gopls/internal/tokeninternal/tokeninternal_test.go
similarity index 96%
rename from internal/tokeninternal/tokeninternal_test.go
rename to gopls/internal/tokeninternal/tokeninternal_test.go
index 7fd14fe..9b13a2e 100644
--- a/internal/tokeninternal/tokeninternal_test.go
+++ b/gopls/internal/tokeninternal/tokeninternal_test.go
@@ -10,7 +10,7 @@
"strings"
"testing"
- "golang.org/x/tools/internal/tokeninternal"
+ "golang.org/x/tools/gopls/internal/tokeninternal"
)
func TestAddExistingFiles(t *testing.T) {
diff --git a/internal/tokeninternal/tokeninternal.go b/internal/tokeninternal/tokeninternal.go
index ff9437a..2c8d70a 100644
--- a/internal/tokeninternal/tokeninternal.go
+++ b/internal/tokeninternal/tokeninternal.go
@@ -7,9 +7,7 @@
package tokeninternal
import (
- "fmt"
"go/token"
- "sort"
"sync"
"unsafe"
)
@@ -45,93 +43,3 @@
defer ptr.mu.Unlock()
return ptr.lines
}
-
-// AddExistingFiles adds the specified files to the FileSet if they
-// are not already present. It panics if any pair of files in the
-// resulting FileSet would overlap.
-func AddExistingFiles(fset *token.FileSet, files []*token.File) {
- // Punch through the FileSet encapsulation.
- type tokenFileSet struct {
- // This type remained essentially consistent from go1.16 to go1.21.
- mutex sync.RWMutex
- base int
- files []*token.File
- _ *token.File // changed to atomic.Pointer[token.File] in go1.19
- }
-
- // If the size of token.FileSet changes, this will fail to compile.
- const delta = int64(unsafe.Sizeof(tokenFileSet{})) - int64(unsafe.Sizeof(token.FileSet{}))
- var _ [-delta * delta]int
-
- type uP = unsafe.Pointer
- var ptr *tokenFileSet
- *(*uP)(uP(&ptr)) = uP(fset)
- ptr.mutex.Lock()
- defer ptr.mutex.Unlock()
-
- // Merge and sort.
- newFiles := append(ptr.files, files...)
- sort.Slice(newFiles, func(i, j int) bool {
- return newFiles[i].Base() < newFiles[j].Base()
- })
-
- // Reject overlapping files.
- // Discard adjacent identical files.
- out := newFiles[:0]
- for i, file := range newFiles {
- if i > 0 {
- prev := newFiles[i-1]
- if file == prev {
- continue
- }
- if prev.Base()+prev.Size()+1 > file.Base() {
- panic(fmt.Sprintf("file %s (%d-%d) overlaps with file %s (%d-%d)",
- prev.Name(), prev.Base(), prev.Base()+prev.Size(),
- file.Name(), file.Base(), file.Base()+file.Size()))
- }
- }
- out = append(out, file)
- }
- newFiles = out
-
- ptr.files = newFiles
-
- // Advance FileSet.Base().
- if len(newFiles) > 0 {
- last := newFiles[len(newFiles)-1]
- newBase := last.Base() + last.Size() + 1
- if ptr.base < newBase {
- ptr.base = newBase
- }
- }
-}
-
-// FileSetFor returns a new FileSet containing a sequence of new Files with
-// the same base, size, and line as the input files, for use in APIs that
-// require a FileSet.
-//
-// Precondition: the input files must be non-overlapping, and sorted in order
-// of their Base.
-func FileSetFor(files ...*token.File) *token.FileSet {
- fset := token.NewFileSet()
- for _, f := range files {
- f2 := fset.AddFile(f.Name(), f.Base(), f.Size())
- lines := GetLines(f)
- f2.SetLines(lines)
- }
- return fset
-}
-
-// CloneFileSet creates a new FileSet holding all files in fset. It does not
-// create copies of the token.Files in fset: they are added to the resulting
-// FileSet unmodified.
-func CloneFileSet(fset *token.FileSet) *token.FileSet {
- var files []*token.File
- fset.Iterate(func(f *token.File) bool {
- files = append(files, f)
- return true
- })
- newFileSet := token.NewFileSet()
- AddExistingFiles(newFileSet, files)
- return newFileSet
-}
