gopls/api-diff: create api-diff command for gopls api

This change adds a command that can be used to see the difference in
API between two gopls versions. It prints out the changes in a way that
can be copy-pasted into the release notes.

Also, only run the copyright test with 1.18. I wanted to do this before
to use filepath.WalkDir, but now it also doesn't work with generic
syntax (it doesn't use packages.Load, so doesn't respect build tags).

Fixes golang/go#46652

Change-Id: I3670e0289a8eeaca02f4dcd8f88f206796ed2462
Reviewed-on: https://go-review.googlesource.com/c/tools/+/327276
Trust: Rebecca Stambler <rstambler@golang.org>
Run-TryBot: Rebecca Stambler <rstambler@golang.org>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

copyright/copyright.go

diff --git a/copyright/copyright.go b/copyright/copyright.go
index a20d623..4a04d13 100644
--- a/copyright/copyright.go
+++ b/copyright/copyright.go
@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:build go1.18
+// +build go1.18
+
 // Package copyright checks that files have the correct copyright notices.
 package copyright
 
@@ -9,8 +12,8 @@
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"io/fs"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -18,13 +21,18 @@
 
 func checkCopyright(dir string) ([]string, error) {
 	var files []string
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		if info.IsDir() {
+		if d.IsDir() {
 			// Skip directories like ".git".
-			if strings.HasPrefix(info.Name(), ".") {
+			if strings.HasPrefix(d.Name(), ".") {
+				return filepath.SkipDir
+			}
+			// Skip any directory that starts with an underscore, as the go
+			// command would.
+			if strings.HasPrefix(d.Name(), "_") {
 				return filepath.SkipDir
 			}
 			return nil

copyright/copyright_test.go

diff --git a/copyright/copyright_test.go b/copyright/copyright_test.go
index bfab43c..1d63147 100644
--- a/copyright/copyright_test.go
+++ b/copyright/copyright_test.go
@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:build go1.18
+// +build go1.18
+
 package copyright
 
 import (

gopls/api-diff/api_diff.go

diff --git a/gopls/api-diff/api_diff.go b/gopls/api-diff/api_diff.go
new file mode 100644
index 0000000..1b98a64
--- /dev/null
+++ b/gopls/api-diff/api_diff.go
@@ -0,0 +1,264 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.18
+// +build go1.18
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	difflib "golang.org/x/tools/internal/lsp/diff"
+	"golang.org/x/tools/internal/lsp/diff/myers"
+	"golang.org/x/tools/internal/lsp/source"
+)
+
+var (
+	previousVersionFlag = flag.String("prev", "", "version to compare against")
+	versionFlag         = flag.String("version", "", "version being tagged, or current version if omitted")
+)
+
+func main() {
+	flag.Parse()
+
+	apiDiff, err := diffAPI(*versionFlag, *previousVersionFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf(`
+%s
+`, apiDiff)
+}
+
+type JSON interface {
+	String() string
+	Write(io.Writer)
+}
+
+func diffAPI(version, prev string) (string, error) {
+	previousApi, err := loadAPI(prev)
+	if err != nil {
+		return "", err
+	}
+	var currentApi *source.APIJSON
+	if version == "" {
+		currentApi = source.GeneratedAPIJSON
+	} else {
+		var err error
+		currentApi, err = loadAPI(version)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	b := &strings.Builder{}
+	if err := diff(b, previousApi.Commands, currentApi.Commands, "command", func(c *source.CommandJSON) string {
+		return c.Command
+	}, diffCommands); err != nil {
+		return "", err
+	}
+	if diff(b, previousApi.Analyzers, currentApi.Analyzers, "analyzer", func(a *source.AnalyzerJSON) string {
+		return a.Name
+	}, diffAnalyzers); err != nil {
+		return "", err
+	}
+	if err := diff(b, previousApi.Lenses, currentApi.Lenses, "code lens", func(l *source.LensJSON) string {
+		return l.Lens
+	}, diffLenses); err != nil {
+		return "", err
+	}
+	for key, prev := range previousApi.Options {
+		current, ok := currentApi.Options[key]
+		if !ok {
+			panic(fmt.Sprintf("unexpected option key: %s", key))
+		}
+		if err := diff(b, prev, current, "option", func(o *source.OptionJSON) string {
+			return o.Name
+		}, diffOptions); err != nil {
+			return "", err
+		}
+	}
+
+	return b.String(), nil
+}
+
+func diff[T JSON](b *strings.Builder, previous, new []T, kind string, uniqueKey func(T) string, diffFunc func(*strings.Builder, T, T)) error {
+	prevJSON := collect(previous, uniqueKey)
+	newJSON := collect(new, uniqueKey)
+	for k := range newJSON {
+		delete(prevJSON, k)
+	}
+	for _, deleted := range prevJSON {
+		b.WriteString(fmt.Sprintf("%s %s was deleted.\n", kind, deleted))
+	}
+	for _, prev := range previous {
+		delete(newJSON, uniqueKey(prev))
+	}
+	if len(newJSON) > 0 {
+		b.WriteString("The following commands were added:\n")
+		for _, n := range newJSON {
+			n.Write(b)
+			b.WriteByte('\n')
+		}
+	}
+	previousMap := collect(previous, uniqueKey)
+	for _, current := range new {
+		prev, ok := previousMap[uniqueKey(current)]
+		if !ok {
+			continue
+		}
+		c, p := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
+		prev.Write(p)
+		current.Write(c)
+		if diff, err := diffStr(p.String(), c.String()); err == nil && diff != "" {
+			diffFunc(b, prev, current)
+			b.WriteString("\n--\n")
+		}
+	}
+	return nil
+}
+
+func collect[T JSON](args []T, uniqueKey func(T) string) map[string]T {
+	m := map[string]T{}
+	for _, arg := range args {
+		m[uniqueKey(arg)] = arg
+	}
+	return m
+}
+
+func loadAPI(version string) (*source.APIJSON, error) {
+	dir, err := ioutil.TempDir("", "gopath*")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(dir)
+
+	if err := os.Mkdir(fmt.Sprintf("%s/src", dir), 0776); err != nil {
+		return nil, err
+	}
+	goCmd, err := exec.LookPath("go")
+	if err != nil {
+		return nil, err
+	}
+	cmd := exec.Cmd{
+		Path: goCmd,
+		Args: []string{"go", "get", fmt.Sprintf("golang.org/x/tools/gopls@%s", version)},
+		Dir:  dir,
+		Env:  append(os.Environ(), fmt.Sprintf("GOPATH=%s", dir)),
+	}
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+	cmd = exec.Cmd{
+		Path: filepath.Join(dir, "bin", "gopls"),
+		Args: []string{"gopls", "api-json"},
+		Dir:  dir,
+	}
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	apiJson := &source.APIJSON{}
+	if err := json.Unmarshal(out, apiJson); err != nil {
+		return nil, err
+	}
+	return apiJson, nil
+}
+
+func diffCommands(b *strings.Builder, prev, current *source.CommandJSON) {
+	if prev.Title != current.Title {
+		b.WriteString(fmt.Sprintf("Title changed from %q to %q\n", prev.Title, current.Title))
+	}
+	if prev.Doc != current.Doc {
+		b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", prev.Doc, current.Doc))
+	}
+	if prev.ArgDoc != current.ArgDoc {
+		b.WriteString("Arguments changed from " + formatBlock(prev.ArgDoc) + " to " + formatBlock(current.ArgDoc))
+	}
+	if prev.ResultDoc != current.ResultDoc {
+		b.WriteString("Results changed from " + formatBlock(prev.ResultDoc) + " to " + formatBlock(current.ResultDoc))
+	}
+}
+
+func diffAnalyzers(b *strings.Builder, previous, current *source.AnalyzerJSON) {
+	b.WriteString(fmt.Sprintf("Changes to analyzer %s:\n\n", current.Name))
+	if previous.Doc != current.Doc {
+		b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", previous.Doc, current.Doc))
+	}
+	if previous.Default != current.Default {
+		b.WriteString(fmt.Sprintf("Default changed from %v to %v\n", previous.Default, current.Default))
+	}
+}
+
+func diffLenses(b *strings.Builder, previous, current *source.LensJSON) {
+	b.WriteString(fmt.Sprintf("Changes to code lens %s:\n\n", current.Title))
+	if previous.Title != current.Title {
+		b.WriteString(fmt.Sprintf("Title changed from %q to %q\n", previous.Title, current.Title))
+	}
+	if previous.Doc != current.Doc {
+		b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", previous.Doc, current.Doc))
+	}
+}
+
+func diffOptions(b *strings.Builder, previous, current *source.OptionJSON) {
+	b.WriteString(fmt.Sprintf("Changes to option %s:\n\n", current.Name))
+	if previous.Doc != current.Doc {
+		diff, err := diffStr(previous.Doc, current.Doc)
+		if err != nil {
+			panic(err)
+		}
+		b.WriteString(fmt.Sprintf("Documentation changed:\n%s\n", diff))
+	}
+	if previous.Default != current.Default {
+		b.WriteString(fmt.Sprintf("Default changed from %q to %q\n", previous.Default, current.Default))
+	}
+	if previous.Hierarchy != current.Hierarchy {
+		b.WriteString(fmt.Sprintf("Categorization changed from %q to %q\n", previous.Hierarchy, current.Hierarchy))
+	}
+	if previous.Status != current.Status {
+		b.WriteString(fmt.Sprintf("Status changed from %q to %q\n", previous.Status, current.Status))
+	}
+	if previous.Type != current.Type {
+		b.WriteString(fmt.Sprintf("Type changed from %q to %q\n", previous.Type, current.Type))
+	}
+	// TODO(rstambler): Handle possibility of same number but different keys/values.
+	if len(previous.EnumKeys.Keys) != len(current.EnumKeys.Keys) {
+		b.WriteString(fmt.Sprintf("Enum keys changed from\n%s\n to \n%s\n", previous.EnumKeys, current.EnumKeys))
+	}
+	if len(previous.EnumValues) != len(current.EnumValues) {
+		b.WriteString(fmt.Sprintf("Enum values changed from\n%s\n to \n%s\n", previous.EnumValues, current.EnumValues))
+	}
+}
+
+func formatBlock(str string) string {
+	if str == "" {
+		return `""`
+	}
+	return "\n```\n" + str + "\n```\n"
+}
+
+func diffStr(before, after string) (string, error) {
+	// Add newlines to avoid newline messages in diff.
+	if before == after {
+		return "", nil
+	}
+	before += "\n"
+	after += "\n"
+	d, err := myers.ComputeEdits("", before, after)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%q", difflib.ToUnified("previous", "current", before, d)), err
+}

gopls/doc/generate.go

diff --git a/gopls/doc/generate.go b/gopls/doc/generate.go
index 91d45ba..b6153e1 100644
--- a/gopls/doc/generate.go
+++ b/gopls/doc/generate.go
@@ -370,7 +370,6 @@
 }
 
 func loadCommands(pkg *packages.Package) ([]*source.CommandJSON, error) {
-
 	var commands []*source.CommandJSON
 
 	_, cmds, err := commandmeta.Load()
@@ -553,8 +552,6 @@
 	return buf.Bytes(), nil
 }
 
-var parBreakRE = regexp.MustCompile("\n{2,}")
-
 type optionsGroup struct {
 	title   string
 	final   string
@@ -583,10 +580,8 @@
 			writeTitle(section, h.final, level)
 			for _, opt := range h.options {
 				header := strMultiply("#", level+1)
-				fmt.Fprintf(section, "%s **%v** *%v*\n\n", header, opt.Name, opt.Type)
-				writeStatus(section, opt.Status)
-				enumValues := collectEnums(opt)
-				fmt.Fprintf(section, "%v%v\nDefault: `%v`.\n\n", opt.Doc, enumValues, opt.Default)
+				section.Write([]byte(fmt.Sprintf("%s ", header)))
+				opt.Write(section)
 			}
 		}
 		var err error
@@ -657,38 +652,6 @@
 	return groups
 }
 
-func collectEnums(opt *source.OptionJSON) string {
-	var b strings.Builder
-	write := func(name, doc string, index, len int) {
-		if doc != "" {
-			unbroken := parBreakRE.ReplaceAllString(doc, "\\\n")
-			fmt.Fprintf(&b, "* %s", unbroken)
-		} else {
-			fmt.Fprintf(&b, "* `%s`", name)
-		}
-		if index < len-1 {
-			fmt.Fprint(&b, "\n")
-		}
-	}
-	if len(opt.EnumValues) > 0 && opt.Type == "enum" {
-		b.WriteString("\nMust be one of:\n\n")
-		for i, val := range opt.EnumValues {
-			write(val.Value, val.Doc, i, len(opt.EnumValues))
-		}
-	} else if len(opt.EnumKeys.Keys) > 0 && shouldShowEnumKeysInSettings(opt.Name) {
-		b.WriteString("\nCan contain any of:\n\n")
-		for i, val := range opt.EnumKeys.Keys {
-			write(val.Name, val.Doc, i, len(opt.EnumKeys.Keys))
-		}
-	}
-	return b.String()
-}
-
-func shouldShowEnumKeysInSettings(name string) bool {
-	// Both of these fields have too many possible options to print.
-	return !hardcodedEnumKeys(name)
-}
-
 func hardcodedEnumKeys(name string) bool {
 	return name == "analyses" || name == "codelenses"
 }
@@ -710,20 +673,6 @@
 	fmt.Fprintf(w, "%s %s\n\n", strMultiply("#", level), capitalize(title))
 }
 
-func writeStatus(section io.Writer, status string) {
-	switch status {
-	case "":
-	case "advanced":
-		fmt.Fprint(section, "**This is an advanced setting and should not be configured by most `gopls` users.**\n\n")
-	case "debug":
-		fmt.Fprint(section, "**This setting is for debugging purposes only.**\n\n")
-	case "experimental":
-		fmt.Fprint(section, "**This setting is experimental and may be deleted.**\n\n")
-	default:
-		fmt.Fprintf(section, "**Status: %s.**\n\n", status)
-	}
-}
-
 func capitalize(s string) string {
 	return string(unicode.ToUpper(rune(s[0]))) + s[1:]
 }
@@ -739,13 +688,7 @@
 func rewriteCommands(doc []byte, api *source.APIJSON) ([]byte, error) {
 	section := bytes.NewBuffer(nil)
 	for _, command := range api.Commands {
-		fmt.Fprintf(section, "### **%v**\nIdentifier: `%v`\n\n%v\n\n", command.Title, command.Command, command.Doc)
-		if command.ArgDoc != "" {
-			fmt.Fprintf(section, "Args:\n\n```\n%s\n```\n\n", command.ArgDoc)
-		}
-		if command.ResultDoc != "" {
-			fmt.Fprintf(section, "Result:\n\n```\n%s\n```\n\n", command.ResultDoc)
-		}
+		command.Write(section)
 	}
 	return replaceSection(doc, "Commands", section.Bytes())
 }

gopls/go.mod

diff --git a/gopls/go.mod b/gopls/go.mod
index f84e502..127b722 100644
--- a/gopls/go.mod
+++ b/gopls/go.mod
@@ -1,6 +1,6 @@
 module golang.org/x/tools/gopls
 
-go 1.17
+go 1.18
 
 require (
 	github.com/BurntSushi/toml v0.4.1 // indirect

internal/lsp/source/options.go

diff --git a/internal/lsp/source/options.go b/internal/lsp/source/options.go
index 2594cb4..9bc73a9 100644
--- a/internal/lsp/source/options.go
+++ b/internal/lsp/source/options.go
@@ -7,6 +7,7 @@
 import (
 	"context"
 	"fmt"
+	"io"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -1280,6 +1281,69 @@
 	Hierarchy  string
 }
 
+func (o *OptionJSON) String() string {
+	return o.Name
+}
+
+func (o *OptionJSON) Write(w io.Writer) {
+	fmt.Fprintf(w, "**%v** *%v*\n\n", o.Name, o.Type)
+	writeStatus(w, o.Status)
+	enumValues := collectEnums(o)
+	fmt.Fprintf(w, "%v%v\nDefault: `%v`.\n\n", o.Doc, enumValues, o.Default)
+}
+
+func writeStatus(section io.Writer, status string) {
+	switch status {
+	case "":
+	case "advanced":
+		fmt.Fprint(section, "**This is an advanced setting and should not be configured by most `gopls` users.**\n\n")
+	case "debug":
+		fmt.Fprint(section, "**This setting is for debugging purposes only.**\n\n")
+	case "experimental":
+		fmt.Fprint(section, "**This setting is experimental and may be deleted.**\n\n")
+	default:
+		fmt.Fprintf(section, "**Status: %s.**\n\n", status)
+	}
+}
+
+var parBreakRE = regexp.MustCompile("\n{2,}")
+
+func collectEnums(opt *OptionJSON) string {
+	var b strings.Builder
+	write := func(name, doc string, index, len int) {
+		if doc != "" {
+			unbroken := parBreakRE.ReplaceAllString(doc, "\\\n")
+			fmt.Fprintf(&b, "* %s", unbroken)
+		} else {
+			fmt.Fprintf(&b, "* `%s`", name)
+		}
+		if index < len-1 {
+			fmt.Fprint(&b, "\n")
+		}
+	}
+	if len(opt.EnumValues) > 0 && opt.Type == "enum" {
+		b.WriteString("\nMust be one of:\n\n")
+		for i, val := range opt.EnumValues {
+			write(val.Value, val.Doc, i, len(opt.EnumValues))
+		}
+	} else if len(opt.EnumKeys.Keys) > 0 && shouldShowEnumKeysInSettings(opt.Name) {
+		b.WriteString("\nCan contain any of:\n\n")
+		for i, val := range opt.EnumKeys.Keys {
+			write(val.Name, val.Doc, i, len(opt.EnumKeys.Keys))
+		}
+	}
+	return b.String()
+}
+
+func shouldShowEnumKeysInSettings(name string) bool {
+	// Both of these fields have too many possible options to print.
+	return !hardcodedEnumKeys(name)
+}
+
+func hardcodedEnumKeys(name string) bool {
+	return name == "analyses" || name == "codelenses"
+}
+
 type EnumKeys struct {
 	ValueType string
 	Keys      []EnumKey
@@ -1304,14 +1368,44 @@
 	ResultDoc string
 }
 
+func (c *CommandJSON) String() string {
+	return c.Command
+}
+
+func (c *CommandJSON) Write(w io.Writer) {
+	fmt.Fprintf(w, "### **%v**\nIdentifier: `%v`\n\n%v\n\n", c.Title, c.Command, c.Doc)
+	if c.ArgDoc != "" {
+		fmt.Fprintf(w, "Args:\n\n```\n%s\n```\n\n", c.ArgDoc)
+	}
+	if c.ResultDoc != "" {
+		fmt.Fprintf(w, "Result:\n\n```\n%s\n```\n\n", c.ResultDoc)
+	}
+}
+
 type LensJSON struct {
 	Lens  string
 	Title string
 	Doc   string
 }
 
+func (l *LensJSON) String() string {
+	return l.Title
+}
+
+func (l *LensJSON) Write(w io.Writer) {
+	fmt.Fprintf(w, "%s (%s): %s", l.Title, l.Lens, l.Doc)
+}
+
 type AnalyzerJSON struct {
 	Name    string
 	Doc     string
 	Default bool
 }
+
+func (a *AnalyzerJSON) String() string {
+	return a.Name
+}
+
+func (a *AnalyzerJSON) Write(w io.Writer) {
+	fmt.Fprintf(w, "%s (%s): %v", a.Name, a.Doc, a.Default)
+}