cmd/gomote/push.go - build - Git at Google

 // Copyright 2015 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package main

 import (
 	"archive/tar"
 	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
 	"crypto/sha1"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"

 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/internal/gomote/protos"
 	"golang.org/x/sync/errgroup"
 )

 func push(args []string) error {
 	fs := flag.NewFlagSet("push", flag.ContinueOnError)
 	var dryRun bool
 	fs.BoolVar(&dryRun, "dry-run", false, "print what would be done only")
 	fs.Usage = func() {
 		fmt.Fprintln(os.Stderr, "push usage: gomote push <instance>")
 		fs.PrintDefaults()
 		os.Exit(1)
 	}
 	fs.Parse(args)

 	goroot, err := getGOROOT()
 	if err != nil {
 		return err
 	}

 	var pushSet []string
 	if fs.NArg() == 1 {
 		pushSet = append(pushSet, fs.Arg(0))
 	} else if activeGroup != nil {
 		for _, inst := range activeGroup.Instances {
 			pushSet = append(pushSet, inst)
 		}
 	} else {
 		fs.Usage()
 	}

 	detailedProgress := len(pushSet) == 1
 	eg, ctx := errgroup.WithContext(context.Background())
 	for _, inst := range pushSet {
 		inst := inst
 		eg.Go(func() error {
 			fmt.Fprintf(os.Stderr, "# Pushing GOROOT %q to %q...\n", goroot, inst)
 			return doPush(ctx, inst, goroot, dryRun, detailedProgress)
 		})
 	}
 	return eg.Wait()
 }

 func doPush(ctx context.Context, name, goroot string, dryRun, detailedProgress bool) error {
 	logf := func(s string, a ...interface{}) {
 		if detailedProgress {
 			log.Printf(s, a...)
 		}
 	}
 	haveGo14 := false
 	remote := map[string]buildlet.DirEntry{} // keys like "src/make.bash"

 	client := gomoteServerClient(ctx)
 	resp, err := client.ListDirectory(ctx, &protos.ListDirectoryRequest{
 		GomoteId:  name,
 		Directory: ".",
 		Recursive: true,
 		SkipFiles: []string{
 			// Ignore binary output directories:
 			"go/pkg", "go/bin",
 			// We don't care about the digest of
 			// particular source files for Go 1.4.  And
 			// exclude /pkg. This leaves go1.4/bin, which
 			// is enough to know whether we have Go 1.4 or
 			// not.
 			"go1.4/src", "go1.4/pkg",
 			// Ignore the cache and tmp directories, these slowly grow, and will
 			// eventually cause the listing to exceed the maximum gRPC message
 			// size.
 			"gocache", "goplscache", "tmp",
 		},
 		Digest: true,
 	})
 	if err != nil {
 		return fmt.Errorf("error listing buildlet's existing files: %w", err)
 	}
 	for _, entry := range resp.GetEntries() {
 		de := buildlet.DirEntry{Line: entry}
 		en := de.Name()
 		if strings.HasPrefix(en, "go1.4/") {
 			haveGo14 = true
 			continue
 		}
 		if strings.HasPrefix(en, "go/") && en != "go/" {
 			remote[en[len("go/"):]] = de
 		}
 	}
 	if !haveGo14 {
 		logf("installing go1.4")
 		if dryRun {
 			logf("(Dry-run) Would have pushed go1.4")
 		} else {
 			_, err := client.AddBootstrap(ctx, &protos.AddBootstrapRequest{
 				GomoteId: name,
 			})
 			if err != nil {
 				return fmt.Errorf("unable to add bootstrap version of Go to instance: %w", err)
 			}
 		}
 	}

 	// Invoke 'git check-ignore' and use it to query whether paths have been gitignored.
 	// If anything goes wrong at any point, fall back to assuming that nothing is gitignored.
 	var isGitIgnored func(string) bool
 	gci := exec.Command("git", "-C", goroot, "check-ignore", "--stdin", "-n", "-v", "-z")
 	gci.Env = append(os.Environ(), "GIT_FLUSH=1")
 	gciIn, errIn := gci.StdinPipe()
 	defer gciIn.Close() // allow git process to exit
 	gciOut, errOut := gci.StdoutPipe()
 	gciErr := &bytes.Buffer{}
 	gci.Stderr = gciErr
 	errStart := gci.Start()
 	if errIn != nil || errOut != nil || errStart != nil {
 		isGitIgnored = func(string) bool { return false }
 	} else {
 		var failed bool
 		br := bufio.NewReader(gciOut)
 		isGitIgnored = func(path string) bool {
 			if failed {
 				return false
 			}
 			fmt.Fprintf(gciIn, "%s\x00", path)
 			// Response is of form "<source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname> <NULL>"
 			// Read all four and check that the path is correct.
 			// If so, the path is ignored iff the source (reason why ignored) is non-empty.
 			var resp [4][]byte
 			for i := range resp {
 				b, err := br.ReadBytes(0)
 				if err != nil {
 					gci.Wait()
 					logf("git check-ignore %q exited unexpectedly:\n%v", path, gciErr.String())
 					failed = true
 					return false
 				}
 				resp[i] = b[:len(b)-1] // drop trailing NULL
 			}
 			// Sanity check
 			if string(resp[3]) != path {
 				panic("git check-ignore path did not roundtrip, got " + string(resp[3]) + " sent " + path)
 			}
 			return len(resp[0]) > 0
 		}
 	}

 	type fileInfo struct {
 		fi   os.FileInfo
 		sha1 string // if regular file
 	}
 	local := map[string]fileInfo{} // keys like "src/make.bash"

 	// Ensure that the goroot passed to filepath.Walk ends in a trailing slash,
 	// so that if GOROOT is a symlink we walk the underlying directory.
 	walkRoot := goroot
 	if walkRoot != "" && !os.IsPathSeparator(walkRoot[len(walkRoot)-1]) {
 		walkRoot += string(filepath.Separator)
 	}
 	if err := filepath.Walk(walkRoot, func(path string, fi os.FileInfo, err error) error {
 		if isEditorBackup(path) {
 			return nil
 		}
 		if err != nil {
 			return err
 		}
 		rel, err := filepath.Rel(goroot, path)
 		if err != nil {
 			return fmt.Errorf("error calculating relative path from %q to %q", goroot, path)
 		}
 		rel = filepath.ToSlash(rel)
 		if rel == "." {
 			return nil
 		}
 		if rel == ".git" {
 			if fi.IsDir() {
 				return filepath.SkipDir
 			}
 			return nil // .git is a file in `git worktree` checkouts.
 		}
 		if fi.IsDir() {
 			switch rel {
 			case "pkg", "bin":
 				return filepath.SkipDir
 			}
 		}
 		inf := fileInfo{fi: fi}
 		if isGitIgnored(path) {
 			if fi.IsDir() {
 				return filepath.SkipDir
 			}
 			return nil
 		}
 		if fi.Mode().IsRegular() {
 			inf.sha1, err = fileSHA1(path)
 			if err != nil {
 				return err
 			}
 		}
 		local[rel] = inf
 		return nil
 	}); err != nil {
 		return fmt.Errorf("error enumerating local GOROOT files: %w", err)
 	}

 	var toDel []string
 	for rel := range remote {
 		if rel == "VERSION" {
 			// Don't delete this. It's harmless, and
 			// necessary. Clients can overwrite it if they
 			// want. But if there's no VERSION file there,
 			// make.bash/bat assumes there's a git repo in
 			// place, but there's not only not a git repo
 			// there with gomote, but there's no git tool
 			// available either.
 			continue
 		}
 		// Also don't delete the auto-generated files from cmd/dist.
 		// Otherwise gomote users can't gomote push + gomote run make.bash
 		// and then iteratively:
 		// -- hack locally
 		// -- gomote push
 		// -- gomote run go test -v ...
 		// Because the go test would fail remotely without
 		// these files if they were deleted by gomote push.
 		if isGoToolDistGenerated(rel) {
 			continue
 		}
 		if isGitIgnored(rel) {
 			// Don't delete remote gitignored files; this breaks built toolchains.
 			continue
 		}
 		rel = strings.TrimRight(rel, "/")
 		if rel == "" {
 			continue
 		}
 		if _, ok := local[rel]; !ok {
 			toDel = append(toDel, rel)
 		}
 	}
 	if len(toDel) > 0 {
 		withGo := make([]string, len(toDel)) // with the "go/" prefix
 		for i, v := range toDel {
 			withGo[i] = "go/" + v
 		}
 		sort.Strings(withGo)
 		if dryRun {
 			logf("(Dry-run) Would have deleted remote files: %q", withGo)
 		} else {
 			logf("Deleting remote files: %q", withGo)
 			if _, err := client.RemoveFiles(ctx, &protos.RemoveFilesRequest{
 				GomoteId: name,
 				Paths:    withGo,
 			}); err != nil {
 				return fmt.Errorf("failed to delete remote unwanted files: %w", err)
 			}
 		}
 	}
 	var toSend []string
 	notHave := 0
 	const maxNotHavePrint = 5
 	for rel, inf := range local {
 		if isGoToolDistGenerated(rel) || rel == "VERSION.cache" {
 			continue
 		}
 		if !inf.fi.Mode().IsRegular() {
 			if !inf.fi.IsDir() {
 				logf("Ignoring local non-regular, non-directory file %s: %v", rel, inf.fi.Mode())
 			}
 			continue
 		}
 		rem, ok := remote[rel]
 		if !ok {
 			if notHave++; notHave <= maxNotHavePrint {
 				logf("Remote doesn't have %q", rel)
 			}
 			toSend = append(toSend, rel)
 			continue
 		}
 		if rem.Digest() != inf.sha1 {
 			logf("Remote's %s digest is %q; want %q", rel, rem.Digest(), inf.sha1)
 			toSend = append(toSend, rel)
 		}
 	}
 	if notHave > maxNotHavePrint {
 		logf("Remote doesn't have %d files (only showed %d).", notHave, maxNotHavePrint)
 	}
 	_, localHasVersion := local["VERSION"]
 	if _, remoteHasVersion := remote["VERSION"]; !remoteHasVersion && !localHasVersion {
 		logf("Remote lacks a VERSION file; sending a fake one")
 		toSend = append(toSend, "VERSION")
 	}
 	if len(toSend) > 0 {
 		sort.Strings(toSend)
 		tgz, err := generateDeltaTgz(goroot, toSend)
 		if err != nil {
 			return err
 		}
 		logf("Uploading %d new/changed files; %d byte .tar.gz", len(toSend), tgz.Len())
 		if dryRun {
 			logf("(Dry-run mode; not doing anything.")
 			return nil
 		}
 		resp, err := client.UploadFile(ctx, &protos.UploadFileRequest{})
 		if err != nil {
 			return fmt.Errorf("unable to request credentials for a file upload: %w", err)
 		}
 		if err := uploadToGCS(ctx, resp.GetFields(), tgz, resp.GetObjectName(), resp.GetUrl()); err != nil {
 			return fmt.Errorf("unable to upload file to GCS: %w", err)
 		}
 		if _, err := client.WriteTGZFromURL(ctx, &protos.WriteTGZFromURLRequest{
 			GomoteId:  name,
 			Url:       fmt.Sprintf("%s%s", resp.GetUrl(), resp.GetObjectName()),
 			Directory: "go",
 		}); err != nil {
 			return fmt.Errorf("failed writing tarball to buildlet: %w", err)
 		}
 	}
 	return nil
 }

 func isGoToolDistGenerated(path string) bool {
 	switch path {
 	case "src/cmd/cgo/zdefaultcc.go",
 		"src/cmd/go/internal/cfg/zdefaultcc.go",
 		"src/cmd/go/internal/cfg/zosarch.go",
 		"src/cmd/internal/objabi/zbootstrap.go",
 		"src/go/build/zcgo.go",
 		"src/internal/buildcfg/zbootstrap.go",
 		"src/runtime/internal/sys/zversion.go",
 		"src/time/tzdata/zzipdata.go":
 		return true
 	}
 	return false
 }

 func isEditorBackup(path string) bool {
 	base := filepath.Base(path)
 	if strings.HasPrefix(base, ".") && strings.HasSuffix(base, ".swp") {
 		// vi
 		return true
 	}
 	if strings.HasSuffix(path, "~") || strings.HasSuffix(path, "#") ||
 		strings.HasPrefix(base, "#") || strings.HasPrefix(base, ".#") {
 		// emacs
 		return true
 	}
 	return false
 }

 // file is forward-slash separated
 func generateDeltaTgz(goroot string, files []string) (*bytes.Buffer, error) {
 	var buf bytes.Buffer
 	zw := gzip.NewWriter(&buf)
 	tw := tar.NewWriter(zw)
 	for _, file := range files {
 		// Special.
 		if file == "VERSION" && !localFileExists(filepath.Join(goroot, file)) {
 			// TODO(bradfitz): a dummy VERSION file's contents to make things
 			// happy. Notably it starts with "devel ". Do we care about it
 			// being accurate beyond that?
 			version := "devel gomote.XXXXX"
 			if err := tw.WriteHeader(&tar.Header{
 				Name: "VERSION",
 				Mode: 0644,
 				Size: int64(len(version)),
 			}); err != nil {
 				return nil, err
 			}
 			if _, err := io.WriteString(tw, version); err != nil {
 				return nil, err
 			}
 			continue
 		}
 		f, err := os.Open(filepath.Join(goroot, file))
 		if err != nil {
 			return nil, err
 		}
 		fi, err := f.Stat()
 		if err != nil {
 			f.Close()
 			return nil, err
 		}
 		header, err := tar.FileInfoHeader(fi, "")
 		if err != nil {
 			f.Close()
 			return nil, err
 		}
 		header.Name = file // forward slash
 		if err := tw.WriteHeader(header); err != nil {
 			f.Close()
 			return nil, err
 		}
 		if _, err := io.CopyN(tw, f, header.Size); err != nil {
 			f.Close()
 			return nil, fmt.Errorf("error copying contents of %s: %w", file, err)
 		}
 		f.Close()
 	}
 	if err := tw.Close(); err != nil {
 		return nil, err
 	}
 	if err := zw.Close(); err != nil {
 		return nil, err
 	}

 	return &buf, nil
 }

 func fileSHA1(path string) (string, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return "", err
 	}
 	defer f.Close()
 	s1 := sha1.New()
 	if _, err := io.Copy(s1, f); err != nil {
 		return "", err
 	}
 	return fmt.Sprintf("%x", s1.Sum(nil)), nil
 }

 func getGOROOT() (string, error) {
 	goroot := os.Getenv("GOROOT")
 	if goroot == "" {
 		slurp, err := exec.Command("go", "env", "GOROOT").Output()
 		if err != nil {
 			return "", fmt.Errorf("failed to get GOROOT from go env: %w", err)
 		}
 		goroot = strings.TrimSpace(string(slurp))
 		if goroot == "" {
 			return "", errors.New("Failed to get $GOROOT from environment or go env")
 		}
 	}
 	goroot = filepath.Clean(goroot)
 	return goroot, nil
 }

 func localFileExists(path string) bool {
 	_, err := os.Stat(path)
 	return !os.IsNotExist(err)
 }
	// Copyright 2015 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package main

	import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha1"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/build/buildlet"
	"golang.org/x/build/internal/gomote/protos"
	"golang.org/x/sync/errgroup"
	)

	func push(args []string) error {
	fs := flag.NewFlagSet("push", flag.ContinueOnError)
	var dryRun bool
	fs.BoolVar(&dryRun, "dry-run", false, "print what would be done only")
	fs.Usage = func() {
	fmt.Fprintln(os.Stderr, "push usage: gomote push <instance>")
	fs.PrintDefaults()
	os.Exit(1)
	}
	fs.Parse(args)

	goroot, err := getGOROOT()
	if err != nil {
	return err
	}

	var pushSet []string
	if fs.NArg() == 1 {
	pushSet = append(pushSet, fs.Arg(0))
	} else if activeGroup != nil {
	for _, inst := range activeGroup.Instances {
	pushSet = append(pushSet, inst)
	}
	} else {
	fs.Usage()
	}

	detailedProgress := len(pushSet) == 1
	eg, ctx := errgroup.WithContext(context.Background())
	for _, inst := range pushSet {
	inst := inst
	eg.Go(func() error {
	fmt.Fprintf(os.Stderr, "# Pushing GOROOT %q to %q...\n", goroot, inst)
	return doPush(ctx, inst, goroot, dryRun, detailedProgress)
	})
	}
	return eg.Wait()
	}

	func doPush(ctx context.Context, name, goroot string, dryRun, detailedProgress bool) error {
	logf := func(s string, a ...interface{}) {
	if detailedProgress {
	log.Printf(s, a...)
	}
	}
	haveGo14 := false
	remote := map[string]buildlet.DirEntry{} // keys like "src/make.bash"

	client := gomoteServerClient(ctx)
	resp, err := client.ListDirectory(ctx, &protos.ListDirectoryRequest{
	GomoteId: name,
	Directory: ".",
	Recursive: true,
	SkipFiles: []string{
	// Ignore binary output directories:
	"go/pkg", "go/bin",
	// We don't care about the digest of
	// particular source files for Go 1.4. And
	// exclude /pkg. This leaves go1.4/bin, which
	// is enough to know whether we have Go 1.4 or
	// not.
	"go1.4/src", "go1.4/pkg",
	// Ignore the cache and tmp directories, these slowly grow, and will
	// eventually cause the listing to exceed the maximum gRPC message
	// size.
	"gocache", "goplscache", "tmp",
	},
	Digest: true,
	})
	if err != nil {
	return fmt.Errorf("error listing buildlet's existing files: %w", err)
	}
	for _, entry := range resp.GetEntries() {
	de := buildlet.DirEntry{Line: entry}
	en := de.Name()
	if strings.HasPrefix(en, "go1.4/") {
	haveGo14 = true
	continue
	}
	if strings.HasPrefix(en, "go/") && en != "go/" {
	remote[en[len("go/"):]] = de
	}
	}
	if !haveGo14 {
	logf("installing go1.4")
	if dryRun {
	logf("(Dry-run) Would have pushed go1.4")
	} else {
	_, err := client.AddBootstrap(ctx, &protos.AddBootstrapRequest{
	GomoteId: name,
	})
	if err != nil {
	return fmt.Errorf("unable to add bootstrap version of Go to instance: %w", err)
	}
	}
	}

	// Invoke 'git check-ignore' and use it to query whether paths have been gitignored.
	// If anything goes wrong at any point, fall back to assuming that nothing is gitignored.
	var isGitIgnored func(string) bool
	gci := exec.Command("git", "-C", goroot, "check-ignore", "--stdin", "-n", "-v", "-z")
	gci.Env = append(os.Environ(), "GIT_FLUSH=1")
	gciIn, errIn := gci.StdinPipe()
	defer gciIn.Close() // allow git process to exit
	gciOut, errOut := gci.StdoutPipe()
	gciErr := &bytes.Buffer{}
	gci.Stderr = gciErr
	errStart := gci.Start()
	if errIn != nil \|\| errOut != nil \|\| errStart != nil {
	isGitIgnored = func(string) bool { return false }
	} else {
	var failed bool
	br := bufio.NewReader(gciOut)
	isGitIgnored = func(path string) bool {
	if failed {
	return false
	}
	fmt.Fprintf(gciIn, "%s\x00", path)
	// Response is of form "<source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname> <NULL>"
	// Read all four and check that the path is correct.
	// If so, the path is ignored iff the source (reason why ignored) is non-empty.
	var resp [4][]byte
	for i := range resp {
	b, err := br.ReadBytes(0)
	if err != nil {
	gci.Wait()
	logf("git check-ignore %q exited unexpectedly:\n%v", path, gciErr.String())
	failed = true
	return false
	}
	resp[i] = b[:len(b)-1] // drop trailing NULL
	}
	// Sanity check
	if string(resp[3]) != path {
	panic("git check-ignore path did not roundtrip, got " + string(resp[3]) + " sent " + path)
	}
	return len(resp[0]) > 0
	}
	}

	type fileInfo struct {
	fi os.FileInfo
	sha1 string // if regular file
	}
	local := map[string]fileInfo{} // keys like "src/make.bash"

	// Ensure that the goroot passed to filepath.Walk ends in a trailing slash,
	// so that if GOROOT is a symlink we walk the underlying directory.
	walkRoot := goroot
	if walkRoot != "" && !os.IsPathSeparator(walkRoot[len(walkRoot)-1]) {
	walkRoot += string(filepath.Separator)
	}
	if err := filepath.Walk(walkRoot, func(path string, fi os.FileInfo, err error) error {
	if isEditorBackup(path) {
	return nil
	}
	if err != nil {
	return err
	}
	rel, err := filepath.Rel(goroot, path)
	if err != nil {
	return fmt.Errorf("error calculating relative path from %q to %q", goroot, path)
	}
	rel = filepath.ToSlash(rel)
	if rel == "." {
	return nil
	}
	if rel == ".git" {
	if fi.IsDir() {
	return filepath.SkipDir
	}
	return nil // .git is a file in `git worktree` checkouts.
	}
	if fi.IsDir() {
	switch rel {
	case "pkg", "bin":
	return filepath.SkipDir
	}
	}
	inf := fileInfo{fi: fi}
	if isGitIgnored(path) {
	if fi.IsDir() {
	return filepath.SkipDir
	}
	return nil
	}
	if fi.Mode().IsRegular() {
	inf.sha1, err = fileSHA1(path)
	if err != nil {
	return err
	}
	}
	local[rel] = inf
	return nil
	}); err != nil {
	return fmt.Errorf("error enumerating local GOROOT files: %w", err)
	}

	var toDel []string
	for rel := range remote {
	if rel == "VERSION" {
	// Don't delete this. It's harmless, and
	// necessary. Clients can overwrite it if they
	// want. But if there's no VERSION file there,
	// make.bash/bat assumes there's a git repo in
	// place, but there's not only not a git repo
	// there with gomote, but there's no git tool
	// available either.
	continue
	}
	// Also don't delete the auto-generated files from cmd/dist.
	// Otherwise gomote users can't gomote push + gomote run make.bash
	// and then iteratively:
	// -- hack locally
	// -- gomote push
	// -- gomote run go test -v ...
	// Because the go test would fail remotely without
	// these files if they were deleted by gomote push.
	if isGoToolDistGenerated(rel) {
	continue
	}
	if isGitIgnored(rel) {
	// Don't delete remote gitignored files; this breaks built toolchains.
	continue
	}
	rel = strings.TrimRight(rel, "/")
	if rel == "" {
	continue
	}
	if _, ok := local[rel]; !ok {
	toDel = append(toDel, rel)
	}
	}
	if len(toDel) > 0 {
	withGo := make([]string, len(toDel)) // with the "go/" prefix
	for i, v := range toDel {
	withGo[i] = "go/" + v
	}
	sort.Strings(withGo)
	if dryRun {
	logf("(Dry-run) Would have deleted remote files: %q", withGo)
	} else {
	logf("Deleting remote files: %q", withGo)
	if _, err := client.RemoveFiles(ctx, &protos.RemoveFilesRequest{
	GomoteId: name,
	Paths: withGo,
	}); err != nil {
	return fmt.Errorf("failed to delete remote unwanted files: %w", err)
	}
	}
	}
	var toSend []string
	notHave := 0
	const maxNotHavePrint = 5
	for rel, inf := range local {
	if isGoToolDistGenerated(rel) \|\| rel == "VERSION.cache" {
	continue
	}
	if !inf.fi.Mode().IsRegular() {
	if !inf.fi.IsDir() {
	logf("Ignoring local non-regular, non-directory file %s: %v", rel, inf.fi.Mode())
	}
	continue
	}
	rem, ok := remote[rel]
	if !ok {
	if notHave++; notHave <= maxNotHavePrint {
	logf("Remote doesn't have %q", rel)
	}
	toSend = append(toSend, rel)
	continue
	}
	if rem.Digest() != inf.sha1 {
	logf("Remote's %s digest is %q; want %q", rel, rem.Digest(), inf.sha1)
	toSend = append(toSend, rel)
	}
	}
	if notHave > maxNotHavePrint {
	logf("Remote doesn't have %d files (only showed %d).", notHave, maxNotHavePrint)
	}
	_, localHasVersion := local["VERSION"]
	if _, remoteHasVersion := remote["VERSION"]; !remoteHasVersion && !localHasVersion {
	logf("Remote lacks a VERSION file; sending a fake one")
	toSend = append(toSend, "VERSION")
	}
	if len(toSend) > 0 {
	sort.Strings(toSend)
	tgz, err := generateDeltaTgz(goroot, toSend)
	if err != nil {
	return err
	}
	logf("Uploading %d new/changed files; %d byte .tar.gz", len(toSend), tgz.Len())
	if dryRun {
	logf("(Dry-run mode; not doing anything.")
	return nil
	}
	resp, err := client.UploadFile(ctx, &protos.UploadFileRequest{})
	if err != nil {
	return fmt.Errorf("unable to request credentials for a file upload: %w", err)
	}
	if err := uploadToGCS(ctx, resp.GetFields(), tgz, resp.GetObjectName(), resp.GetUrl()); err != nil {
	return fmt.Errorf("unable to upload file to GCS: %w", err)
	}
	if _, err := client.WriteTGZFromURL(ctx, &protos.WriteTGZFromURLRequest{
	GomoteId: name,
	Url: fmt.Sprintf("%s%s", resp.GetUrl(), resp.GetObjectName()),
	Directory: "go",
	}); err != nil {
	return fmt.Errorf("failed writing tarball to buildlet: %w", err)
	}
	}
	return nil
	}

	func isGoToolDistGenerated(path string) bool {
	switch path {
	case "src/cmd/cgo/zdefaultcc.go",
	"src/cmd/go/internal/cfg/zdefaultcc.go",
	"src/cmd/go/internal/cfg/zosarch.go",
	"src/cmd/internal/objabi/zbootstrap.go",
	"src/go/build/zcgo.go",
	"src/internal/buildcfg/zbootstrap.go",
	"src/runtime/internal/sys/zversion.go",
	"src/time/tzdata/zzipdata.go":
	return true
	}
	return false
	}

	func isEditorBackup(path string) bool {
	base := filepath.Base(path)
	if strings.HasPrefix(base, ".") && strings.HasSuffix(base, ".swp") {
	// vi
	return true
	}
	if strings.HasSuffix(path, "~") \|\| strings.HasSuffix(path, "#") \|\|
	strings.HasPrefix(base, "#") \|\| strings.HasPrefix(base, ".#") {
	// emacs
	return true
	}
	return false
	}

	// file is forward-slash separated
	func generateDeltaTgz(goroot string, files []string) (*bytes.Buffer, error) {
	var buf bytes.Buffer
	zw := gzip.NewWriter(&buf)
	tw := tar.NewWriter(zw)
	for _, file := range files {
	// Special.
	if file == "VERSION" && !localFileExists(filepath.Join(goroot, file)) {
	// TODO(bradfitz): a dummy VERSION file's contents to make things
	// happy. Notably it starts with "devel ". Do we care about it
	// being accurate beyond that?
	version := "devel gomote.XXXXX"
	if err := tw.WriteHeader(&tar.Header{
	Name: "VERSION",
	Mode: 0644,
	Size: int64(len(version)),
	}); err != nil {
	return nil, err
	}
	if _, err := io.WriteString(tw, version); err != nil {
	return nil, err
	}
	continue
	}
	f, err := os.Open(filepath.Join(goroot, file))
	if err != nil {
	return nil, err
	}
	fi, err := f.Stat()
	if err != nil {
	f.Close()
	return nil, err
	}
	header, err := tar.FileInfoHeader(fi, "")
	if err != nil {
	f.Close()
	return nil, err
	}
	header.Name = file // forward slash
	if err := tw.WriteHeader(header); err != nil {
	f.Close()
	return nil, err
	}
	if _, err := io.CopyN(tw, f, header.Size); err != nil {
	f.Close()
	return nil, fmt.Errorf("error copying contents of %s: %w", file, err)
	}
	f.Close()
	}
	if err := tw.Close(); err != nil {
	return nil, err
	}
	if err := zw.Close(); err != nil {
	return nil, err
	}

	return &buf, nil
	}

	func fileSHA1(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
	return "", err
	}
	defer f.Close()
	s1 := sha1.New()
	if _, err := io.Copy(s1, f); err != nil {
	return "", err
	}
	return fmt.Sprintf("%x", s1.Sum(nil)), nil
	}

	func getGOROOT() (string, error) {
	goroot := os.Getenv("GOROOT")
	if goroot == "" {
	slurp, err := exec.Command("go", "env", "GOROOT").Output()
	if err != nil {
	return "", fmt.Errorf("failed to get GOROOT from go env: %w", err)
	}
	goroot = strings.TrimSpace(string(slurp))
	if goroot == "" {
	return "", errors.New("Failed to get $GOROOT from environment or go env")
	}
	}
	goroot = filepath.Clean(goroot)
	return goroot, nil
	}

	func localFileExists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
	}