cmd/txtar: add a minimal command for creating txtar archives

I've been working on a design doc for a lazy module loading algorithm,
and I want to be able to present examples as txtar archives (because
that's what the eventual integration tests will be anyway).

But I also want to be able to run the 'go' command interactively in
those archives (to figure out what the behavior today is and how it
would change), and if I decide that I need to change the example it
becomes tedious to edit it locally and then repackage it.

This change adds a minimal binary for constructing and unpacking
archives. The comment section (which is usually relevant to the test)
and the archive itself are passed on stdin and stdout, and files are
read or written relative to the working directory of the command. A
list of files and/or directories can be passed explicitly at creation
time (to bypass extraneous files such as Git metadata).

Since the cmd/go tests in many cases use paths relative to the $WORK
or $GOPATH variables, this binary expands shell variables during both
creation and extraction.

A similar set of utility programs can be found in
github.com/rogpeppe/go-internal/cmd. While those programs are useful,
they are somewhat more flag-intensive, do not support shell-escaped
paths or explicit file lists, and do not extract or accept comment
text on stdio and stdout.

Change-Id: Ibfd2f7b308151b5588bba14c9d66c59453fbdbe0
Reviewed-on: https://go-review.googlesource.com/c/exp/+/218498
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
diff --git a/cmd/txtar/txtar.go b/cmd/txtar/txtar.go
new file mode 100644
index 0000000..e4171dd
--- /dev/null
+++ b/cmd/txtar/txtar.go
@@ -0,0 +1,168 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The txtar command writes or extracts a text-based file archive in the format
+// provided by the golang.org/x/tools/txtar package.
+//
+// The default behavior is to read a comment from stdin and write the archive
+// file containing the recursive contents of the named files and directories,
+// including hidden files, to stdout. Any non-flag arguments to the command name
+// the files and/or directories to include, with the contents of directories
+// included recursively. An empty argument list is equivalent to ".".
+//
+// The --extract (or -x) flag instructs txtar to instead read the archive file
+// from stdin and extract all of its files to corresponding locations relative
+// to the current, writing the archive's comment to stdout.
+//
+// Shell variables in paths are expanded (using os.Expand) if the corresponding
+// variable is set in the process environment. When writing an archive, the
+// variables (before expansion) are preserved in the archived paths.
+//
+// Example usage:
+//
+// 	txtar *.go <README >testdata/example.txt
+//
+// 	txtar --extract <playground_example.txt >main.go
+//
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"golang.org/x/tools/txtar"
+)
+
+var (
+	extractFlag = flag.Bool("extract", false, "if true, extract files from the archive instead of writing to it")
+)
+
+func init() {
+	flag.BoolVar(extractFlag, "x", *extractFlag, "short alias for --extract")
+}
+
+func main() {
+	flag.Parse()
+
+	var err error
+	if *extractFlag {
+		if len(flag.Args()) > 0 {
+			fmt.Fprintln(os.Stderr, "Usage: txtar --extract <archive.txt")
+			os.Exit(2)
+		}
+		err = extract()
+	} else {
+		paths := flag.Args()
+		if len(paths) == 0 {
+			paths = []string{"."}
+		}
+		err = archive(paths)
+	}
+
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func extract() (err error) {
+	b, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		return err
+	}
+
+	ar := txtar.Parse(b)
+	for _, f := range ar.Files {
+		fileName := filepath.FromSlash(path.Clean(expand(f.Name)))
+		if err := os.MkdirAll(filepath.Dir(fileName), 0777); err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(fileName, f.Data, 0666); err != nil {
+			return err
+		}
+	}
+
+	if len(ar.Comment) > 0 {
+		os.Stdout.Write(ar.Comment)
+	}
+	return nil
+}
+
+func archive(paths []string) (err error) {
+	txtarHeader := regexp.MustCompile(`(?m)^-- .* --$`)
+
+	ar := new(txtar.Archive)
+	for _, p := range paths {
+		root := filepath.Clean(expand(p))
+		prefix := root + string(filepath.Separator)
+		err := filepath.Walk(root, func(fileName string, info os.FileInfo, err error) error {
+			if err != nil || info.IsDir() {
+				return err
+			}
+
+			suffix := ""
+			if fileName != root {
+				suffix = strings.TrimPrefix(fileName, prefix)
+			}
+			name := filepath.ToSlash(filepath.Join(p, suffix))
+
+			data, err := ioutil.ReadFile(fileName)
+			if err != nil {
+				return err
+			}
+			if txtarHeader.Match(data) {
+				return fmt.Errorf("cannot archive %s: file contains a txtar header", name)
+			}
+
+			ar.Files = append(ar.Files, txtar.File{Name: name, Data: data})
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	// After we have read all of the source files, read the comment from stdin.
+	//
+	// Wait until the read has been blocked for a while before prompting the user
+	// to enter it: if they are piping the comment in from some other file, the
+	// read should complete very quickly and there is no need for a prompt.
+	// (200ms is typically long enough to read a reasonable comment from the local
+	// machine, but short enough that humans don't notice it.)
+	//
+	// Don't prompt until we have successfully read the other files:
+	// if we encountered an error, we don't need to ask for a comment.
+	timer := time.AfterFunc(200*time.Millisecond, func() {
+		fmt.Fprintln(os.Stderr, "Enter comment:")
+	})
+	comment, err := ioutil.ReadAll(os.Stdin)
+	timer.Stop()
+	if err != nil {
+		return fmt.Errorf("reading comment from %s: %v", os.Stdin.Name(), err)
+	}
+	ar.Comment = bytes.TrimSpace(comment)
+
+	_, err = os.Stdout.Write(txtar.Format(ar))
+	return err
+}
+
+// expand is like os.ExpandEnv, but preserves unescaped variables (instead
+// of escaping them to the empty string) if the variable is not set.
+func expand(p string) string {
+	return os.Expand(p, func(key string) string {
+		v, ok := os.LookupEnv(key)
+		if !ok {
+			return "$" + key
+		}
+		return v
+	})
+}
diff --git a/cmd/txtar/txtar_test.go b/cmd/txtar/txtar_test.go
new file mode 100644
index 0000000..4130336
--- /dev/null
+++ b/cmd/txtar/txtar_test.go
@@ -0,0 +1,120 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+)
+
+const comment = "This is a txtar archive.\n"
+
+const testdata = `This is a txtar archive.
+-- one.txt --
+one
+-- dir/two.txt --
+two
+-- $SPECIAL_LOCATION/three.txt --
+three
+`
+
+func TestMain(m *testing.M) {
+	code := m.Run()
+	txtarBin.once.Do(func() {})
+	if txtarBin.name != "" {
+		os.Remove(txtarBin.name)
+	}
+	os.Exit(code)
+}
+
+func TestRoundTrip(t *testing.T) {
+	os.Setenv("SPECIAL_LOCATION", "special")
+	defer os.Unsetenv("SPECIAL_LOCATION")
+
+	// Expand the testdata archive into a temporary directory.
+	parentDir, err := ioutil.TempDir("", "txtar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(parentDir)
+	dir := filepath.Join(parentDir, "dir")
+	if err := os.Mkdir(dir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if out := txtar(t, dir, testdata, "--extract"); out != comment {
+		t.Fatalf("txtar --extract: stdout:\n%s\nwant:\n%s", out, comment)
+	}
+
+	// Now, re-archive its contents explicitly and ensure that the result matches
+	// the original.
+	args := []string{"one.txt", "dir", "$SPECIAL_LOCATION"}
+	if out := txtar(t, dir, comment, args...); out != testdata {
+		t.Fatalf("txtar %s: archive:\n%s\n\nwant:\n%s", strings.Join(args, " "), out, testdata)
+	}
+}
+
+// txtar runs the txtar command in the given directory with the given input and
+// arguments.
+func txtar(t *testing.T, dir, input string, args ...string) string {
+	t.Helper()
+	cmd := exec.Command(txtarName(t), args...)
+	cmd.Dir = dir
+	cmd.Stdin = strings.NewReader(input)
+	stderr := new(strings.Builder)
+	cmd.Stderr = stderr
+	out, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, stderr)
+	}
+	if stderr.String() != "" {
+		t.Logf("OK: %s\n%s", strings.Join(cmd.Args, " "), stderr)
+	}
+	return string(out)
+}
+
+var txtarBin struct {
+	once sync.Once
+	name string
+	err  error
+}
+
+// txtarName returns the name of the txtar executable, building it if needed.
+func txtarName(t *testing.T) string {
+	t.Helper()
+	if _, err := exec.LookPath("go"); err != nil {
+		t.Skipf("cannot build txtar binary: %v", err)
+	}
+
+	txtarBin.once.Do(func() {
+		exe, err := ioutil.TempFile("", "txtar-*.exe")
+		if err != nil {
+			txtarBin.err = err
+			return
+		}
+		exe.Close()
+		txtarBin.name = exe.Name()
+
+		cmd := exec.Command("go", "build", "-o", txtarBin.name, ".")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			txtarBin.err = fmt.Errorf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, out)
+		}
+	})
+
+	if txtarBin.err != nil {
+		if runtime.GOOS == "android" {
+			t.Skipf("skipping test after failing to build txtar binary: go_android_exec may have failed to copy needed dependencies (see https://golang.org/issue/37088)")
+		}
+		t.Fatal(txtarBin.err)
+	}
+	return txtarBin.name
+}
diff --git a/go.mod b/go.mod
index 84df2a7..bc31db7 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@
 	github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72
 	golang.org/x/image v0.0.0-20190802002840-cff245a6509b
 	golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028
-	golang.org/x/mod v0.1.0
+	golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee
 	golang.org/x/sys v0.0.0-20190412213103-97732733099d
-	golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a
+	golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa
 )
diff --git a/go.sum b/go.sum
index 67aebad..9a50bee 100644
--- a/go.sum
+++ b/go.sum
@@ -5,7 +5,7 @@
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72 h1:b+9H1GAsx5RsjvDFLoS5zkNBzIQMuVKUYQDmxU3N5XE=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -13,8 +13,8 @@
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.1.0 h1:sfUMP1Gu8qASkorDVjnMuvgJzwFbTZSeXFiGBYAVdl4=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -23,6 +23,7 @@
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHqSiGUkO/B4rfyhwqDxDYQ=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa h1:5E4dL8+NgFOgjwbTKz+OOEGGhP+ectTmF842l6KjupQ=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=