cmd/gonew: add new tool for starting a module by copying one
This is an experimental command that perhaps would become "go new"
once we have more experience with it. We want to enable people to
experiment with it and write their own templates and see how it works,
and for that we need to put it in a place where it's reasonable to ask
users to fetch it from. That place is golang.org/x/tools/cmd/gonew.
There is an earlier copy in rsc.io/tmp/gonew, but that isn't the right
place for end users to be fetching something to try.
Once the tool is checked in I intend to start a GitHub discussion
asking for feedback and suggestions about what is missing.
I hope we will be able to identify core functionality that handles a
large fraction of use cases.
I've been using the earlier version myself for a while, and I've found
it very convenient even in other contexts, like I want the code for a
given module and don't want to go look up its Git repo and so on:
go new rsc.io/quote@v1.5.2
cd quote
Change-Id: Ifc27cbd5d87ded89bc707b087b3f08fa70b1ef07
Reviewed-on: https://go-review.googlesource.com/c/tools/+/513737
gopls-CI: kokoro <noreply+kokoro@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Auto-Submit: Russ Cox <rsc@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/cmd/gonew/main.go b/cmd/gonew/main.go
new file mode 100644
index 0000000..920d56a
--- /dev/null
+++ b/cmd/gonew/main.go
@@ -0,0 +1,233 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Gonew starts a new Go module by copying a template module.
+//
+// Usage:
+//
+// gonew srcmod[@version] [dstmod [dir]]
+//
+// Gonew makes a copy of the srcmod module, changing its module path to dstmod.
+// It writes that new module to a new directory named by dir.
+// If dir already exists, it must be an empty directory.
+// If dir is omitted, gonew uses ./elem where elem is the final path element of dstmod.
+//
+// This command is highly experimental and subject to change.
+//
+// # Example
+//
+// To install gonew:
+//
+// go install golang.org/x/tools/cmd/gonew@latest
+//
+// To clone the basic command-line program template golang.org/x/example/hello
+// as your.domain/myprog, in the directory ./myprog:
+//
+// gonew golang.org/x/example/hello your.domain/myprog
+//
+// To clone the latest copy of the rsc.io/quote module, keeping that module path,
+// into ./quote:
+//
+// gonew rsc.io/quote
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "go/parser"
+ "go/token"
+ "io/fs"
+ "log"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "golang.org/x/mod/modfile"
+ "golang.org/x/mod/module"
+ "golang.org/x/tools/internal/edit"
+)
+
+func usage() {
+ fmt.Fprintf(os.Stderr, "usage: gonew srcmod[@version] [dstmod [dir]]\n")
+ fmt.Fprintf(os.Stderr, "See https://pkg.go.dev/golang.org/x/tools/cmd/gonew.\n")
+ os.Exit(2)
+}
+
+func main() {
+ log.SetPrefix("gonew: ")
+ log.SetFlags(0)
+ flag.Usage = usage
+ flag.Parse()
+ args := flag.Args()
+
+ if len(args) < 1 || len(args) > 3 {
+ usage()
+ }
+
+ srcMod := args[0]
+ srcModVers := srcMod
+ if !strings.Contains(srcModVers, "@") {
+ srcModVers += "@latest"
+ }
+ srcMod, _, _ = strings.Cut(srcMod, "@")
+ if err := module.CheckPath(srcMod); err != nil {
+ log.Fatalf("invalid source module name: %v", err)
+ }
+
+ dstMod := srcMod
+ if len(args) >= 2 {
+ dstMod = args[1]
+ if err := module.CheckPath(dstMod); err != nil {
+ log.Fatalf("invalid destination module name: %v", err)
+ }
+ }
+
+ var dir string
+ if len(args) == 3 {
+ dir = args[2]
+ } else {
+ dir = "." + string(filepath.Separator) + path.Base(dstMod)
+ }
+
+ // Dir must not exist or must be an empty directory.
+ de, err := os.ReadDir(dir)
+ if err == nil && len(de) > 0 {
+ log.Fatalf("target directory %s exists and is non-empty", dir)
+ }
+ needMkdir := err != nil
+
+ var stdout, stderr bytes.Buffer
+ cmd := exec.Command("go", "mod", "download", "-json", srcModVers)
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ log.Fatalf("go mod download -json %s: %v\n%s%s", srcModVers, err, stderr.Bytes(), stdout.Bytes())
+ }
+
+ var info struct {
+ Dir string
+ }
+ if err := json.Unmarshal(stdout.Bytes(), &info); err != nil {
+ log.Fatalf("go mod download -json %s: invalid JSON output: %v\n%s%s", srcMod, err, stderr.Bytes(), stdout.Bytes())
+ }
+
+ if needMkdir {
+ if err := os.MkdirAll(dir, 0777); err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ // Copy from module cache into new directory, making edits as needed.
+ filepath.WalkDir(info.Dir, func(src string, d fs.DirEntry, err error) error {
+ if err != nil {
+ log.Fatal(err)
+ }
+ rel, err := filepath.Rel(info.Dir, src)
+ if err != nil {
+ log.Fatal(err)
+ }
+ dst := filepath.Join(dir, rel)
+ if d.IsDir() {
+ if err := os.MkdirAll(dst, 0777); err != nil {
+ log.Fatal(err)
+ }
+ return nil
+ }
+
+ data, err := os.ReadFile(src)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ isRoot := !strings.Contains(rel, string(filepath.Separator))
+ if strings.HasSuffix(rel, ".go") {
+ data = fixGo(data, rel, srcMod, dstMod, isRoot)
+ }
+ if rel == "go.mod" {
+ data = fixGoMod(data, srcMod, dstMod)
+ }
+
+ if err := os.WriteFile(dst, data, 0666); err != nil {
+ log.Fatal(err)
+ }
+ return nil
+ })
+
+ log.Printf("initialized %s in %s", dstMod, dir)
+}
+
+// fixGo rewrites the Go source in data to replace srcMod with dstMod.
+// isRoot indicates whether the file is in the root directory of the module,
+// in which case we also update the package name.
+func fixGo(data []byte, file string, srcMod, dstMod string, isRoot bool) []byte {
+ fset := token.NewFileSet()
+ f, err := parser.ParseFile(fset, file, data, parser.ImportsOnly)
+ if err != nil {
+ log.Fatalf("parsing source module:\n%s", err)
+ }
+
+ buf := edit.NewBuffer(data)
+ at := func(p token.Pos) int {
+ return fset.File(p).Offset(p)
+ }
+
+ srcName := path.Base(srcMod)
+ dstName := path.Base(dstMod)
+ if isRoot {
+ if name := f.Name.Name; name == srcName || name == srcName+"_test" {
+ dname := dstName + strings.TrimPrefix(name, srcName)
+ if !token.IsIdentifier(dname) {
+ log.Fatalf("%s: cannot rename package %s to package %s: invalid package name", file, name, dname)
+ }
+ buf.Replace(at(f.Name.Pos()), at(f.Name.End()), dname)
+ }
+ }
+
+ for _, spec := range f.Imports {
+ path, err := strconv.Unquote(spec.Path.Value)
+ if err != nil {
+ continue
+ }
+ if path == srcMod {
+ if srcName != dstName && spec.Name == nil {
+ // Add package rename because source code uses original name.
+ // The renaming looks strange, but template authors are unlikely to
+ // create a template where the root package is imported by packages
+ // in subdirectories, and the renaming at least keeps the code working.
+ // A more sophisticated approach would be to rename the uses of
+ // the package identifier in the file too, but then you have to worry about
+ // name collisions, and given how unlikely this is, it doesn't seem worth
+ // trying to clean up the file that way.
+ buf.Insert(at(spec.Path.Pos()), srcName+" ")
+ }
+ // Change import path to dstMod
+ buf.Replace(at(spec.Path.Pos()), at(spec.Path.End()), strconv.Quote(dstMod))
+ }
+ if strings.HasPrefix(path, srcMod+"/") {
+ // Change import path to begin with dstMod
+ buf.Replace(at(spec.Path.Pos()), at(spec.Path.End()), strconv.Quote(strings.Replace(path, srcMod, dstMod, 1)))
+ }
+ }
+ return buf.Bytes()
+}
+
+// fixGoMod rewrites the go.mod content in data to replace srcMod with dstMod
+// in the module path.
+func fixGoMod(data []byte, srcMod, dstMod string) []byte {
+ f, err := modfile.ParseLax("go.mod", data, nil)
+ if err != nil {
+ log.Fatalf("parsing source module:\n%s", err)
+ }
+ f.AddModuleStmt(dstMod)
+ new, err := f.Format()
+ if err != nil {
+ return data
+ }
+ return new
+}
diff --git a/cmd/gonew/main_test.go b/cmd/gonew/main_test.go
new file mode 100644
index 0000000..590bda0
--- /dev/null
+++ b/cmd/gonew/main_test.go
@@ -0,0 +1,214 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "archive/zip"
+ "bytes"
+ "fmt"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+
+ "golang.org/x/tools/internal/diffp"
+ "golang.org/x/tools/txtar"
+)
+
+func init() {
+ if os.Getenv("TestGonewMain") == "1" {
+ main()
+ os.Exit(0)
+ }
+}
+
+func Test(t *testing.T) {
+ exe, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Each file in testdata is a txtar file with the command to run,
+ // the contents of modules to initialize in a fake proxy,
+ // the expected stdout and stderr, and the expected file contents.
+ files, err := filepath.Glob("testdata/*.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(files) == 0 {
+ t.Fatal("no test cases")
+ }
+
+ for _, file := range files {
+ t.Run(filepath.Base(file), func(t *testing.T) {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ar := txtar.Parse(data)
+
+ // If the command begins with ! it means it should fail.
+ // After the optional ! the first argument must be 'gonew'
+ // followed by the arguments to gonew.
+ args := strings.Fields(string(ar.Comment))
+ wantFail := false
+ if len(args) > 0 && args[0] == "!" {
+ wantFail = true
+ args = args[1:]
+ }
+ if len(args) == 0 || args[0] != "gonew" {
+ t.Fatalf("invalid command comment")
+ }
+
+ // Collect modules into proxy tree and store in temp directory.
+ dir := t.TempDir()
+ proxyDir := filepath.Join(dir, "proxy")
+ writeProxyFiles(t, proxyDir, ar)
+ extra := ""
+ if runtime.GOOS == "windows" {
+ // Windows absolute paths don't start with / so we need one more.
+ extra = "/"
+ }
+ proxyURL := "file://" + extra + filepath.ToSlash(proxyDir)
+
+ // Run gonew in a fresh 'out' directory.
+ out := filepath.Join(dir, "out")
+ if err := os.Mkdir(out, 0777); err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command(exe, args[1:]...)
+ cmd.Dir = out
+ cmd.Env = append(os.Environ(), "TestGonewMain=1", "GOPROXY="+proxyURL, "GOSUMDB=off")
+ var stdout bytes.Buffer
+ var stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err == nil && wantFail {
+ t.Errorf("unexpected success exit")
+ } else if err != nil && !wantFail {
+ t.Errorf("unexpected failure exit")
+ }
+
+ // Collect the expected output from the txtar.
+ want := make(map[string]txtar.File)
+ for _, f := range ar.Files {
+ if f.Name == "stdout" || f.Name == "stderr" || strings.HasPrefix(f.Name, "out/") {
+ want[f.Name] = f
+ }
+ }
+
+ // Check stdout and stderr.
+ // Change \ to / so Windows output looks like Unix output.
+ stdoutBuf := bytes.ReplaceAll(stdout.Bytes(), []byte(`\`), []byte("/"))
+ stderrBuf := bytes.ReplaceAll(stderr.Bytes(), []byte(`\`), []byte("/"))
+ // Note that stdout and stderr can be omitted from the archive if empty.
+ if !bytes.Equal(stdoutBuf, want["stdout"].Data) {
+ t.Errorf("wrong stdout: %s", diffp.Diff("want", want["stdout"].Data, "have", stdoutBuf))
+ }
+ if !bytes.Equal(stderrBuf, want["stderr"].Data) {
+ t.Errorf("wrong stderr: %s", diffp.Diff("want", want["stderr"].Data, "have", stderrBuf))
+ }
+ delete(want, "stdout")
+ delete(want, "stderr")
+
+ // Check remaining expected outputs.
+ err = filepath.WalkDir(out, func(name string, info fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ data, err := os.ReadFile(name)
+ if err != nil {
+ return err
+ }
+ short := "out" + filepath.ToSlash(strings.TrimPrefix(name, out))
+ f, ok := want[short]
+ if !ok {
+ t.Errorf("unexpected file %s:\n%s", short, data)
+ return nil
+ }
+ delete(want, short)
+ if !bytes.Equal(data, f.Data) {
+ t.Errorf("wrong %s: %s", short, diffp.Diff("want", f.Data, "have", data))
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ for name := range want {
+ t.Errorf("missing file %s", name)
+ }
+ })
+ }
+}
+
+// A Zip is a zip file being written.
+type Zip struct {
+ buf bytes.Buffer
+ w *zip.Writer
+}
+
+// writeProxyFiles collects all the module content from ar and writes
+// files in the format of the proxy URL space, so that the 'proxy' directory
+// can be used in a GOPROXY=file:/// URL.
+func writeProxyFiles(t *testing.T, proxy string, ar *txtar.Archive) {
+ zips := make(map[string]*Zip)
+ others := make(map[string]string)
+ for _, f := range ar.Files {
+ i := strings.Index(f.Name, "@")
+ if i < 0 {
+ continue
+ }
+ j := strings.Index(f.Name[i:], "/")
+ if j < 0 {
+ t.Fatalf("unexpected archive file %s", f.Name)
+ }
+ j += i
+ mod, vers, file := f.Name[:i], f.Name[i+1:j], f.Name[j+1:]
+ zipName := mod + "/@v/" + vers + ".zip"
+ z := zips[zipName]
+ if z == nil {
+ others[mod+"/@v/list"] += vers + "\n"
+ others[mod+"/@v/"+vers+".info"] = fmt.Sprintf("{%q: %q}\n", "Version", vers)
+ z = new(Zip)
+ z.w = zip.NewWriter(&z.buf)
+ zips[zipName] = z
+ }
+ if file == "go.mod" {
+ others[mod+"/@v/"+vers+".mod"] = string(f.Data)
+ }
+ w, err := z.w.Create(f.Name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := w.Write(f.Data); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for name, z := range zips {
+ if err := z.w.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Dir(filepath.Join(proxy, name)), 0777); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(proxy, name), z.buf.Bytes(), 0666); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for name, data := range others {
+ // zip loop already created directory
+ if err := os.WriteFile(filepath.Join(proxy, name), []byte(data), 0666); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
diff --git a/cmd/gonew/testdata/quote.txt b/cmd/gonew/testdata/quote.txt
new file mode 100644
index 0000000..9f166b5
--- /dev/null
+++ b/cmd/gonew/testdata/quote.txt
@@ -0,0 +1,28 @@
+gonew example.com/quote my.com/test
+
+-- example.com/quote@v1.5.2/go.mod --
+module example.com/quote
+-- example.com/quote@v1.5.2/quote.go --
+package quote
+
+import (
+ "example.com/quote/bar"
+)
+
+func Quote() {}
+-- example.com/quote@v1.5.2/quote/another.go --
+package quote // another package quote!
+-- stderr --
+gonew: initialized my.com/test in ./test
+-- out/test/go.mod --
+module my.com/test
+-- out/test/quote.go --
+package test
+
+import (
+ "my.com/test/bar"
+)
+
+func Quote() {}
+-- out/test/quote/another.go --
+package quote // another package quote!
