cmd/gomote: implements GRPC push command
This change adds the implementation for GRPC push command to the
gomote client.
Updates golang/go#48737
For golang/go#47521
Change-Id: Ibb40dff14b9be0c273fb26a625d5e64b1bca25f0
Reviewed-on: https://go-review.googlesource.com/c/build/+/410819
Reviewed-by: Carlos Amedee <carlos@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Auto-Submit: Carlos Amedee <carlos@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Run-TryBot: Carlos Amedee <carlos@golang.org>
diff --git a/cmd/gomote/gomote.go b/cmd/gomote/gomote.go
index 1c8dcf6..65eb3b0 100644
--- a/cmd/gomote/gomote.go
+++ b/cmd/gomote/gomote.go
@@ -155,7 +155,7 @@
registerCommand("ls", "list the contents of a directory on a buildlet", legacyLs)
registerCommand("list", "list active buildlets", legacyList)
registerCommand("ping", "test whether a buildlet is alive and reachable ", ping)
- registerCommand("push", "sync your GOROOT directory to the buildlet", push)
+ registerCommand("push", "sync your GOROOT directory to the buildlet", legacyPush)
registerCommand("put", "put files on a buildlet", legacyPut)
registerCommand("put14", "put Go 1.4 in place", put14)
registerCommand("puttar", "extract a tar.gz to a buildlet", legacyPutTar)
@@ -229,6 +229,7 @@
"put": put,
"puttar": putTar,
"putbootstrap": putBootstrap,
+ "push": push,
}
if len(args) == 0 {
usage()
diff --git a/cmd/gomote/push.go b/cmd/gomote/push.go
index 8aa024b..435261a 100644
--- a/cmd/gomote/push.go
+++ b/cmd/gomote/push.go
@@ -23,9 +23,10 @@
"strings"
"golang.org/x/build/buildlet"
+ "golang.org/x/build/internal/gomote/protos"
)
-func push(args []string) error {
+func legacyPush(args []string) error {
fs := flag.NewFlagSet("push", flag.ContinueOnError)
var dryRun bool
fs.BoolVar(&dryRun, "dry-run", false, "print what would be done only")
@@ -292,6 +293,280 @@
return nil
}
+func push(args []string) error {
+ fs := flag.NewFlagSet("push", flag.ContinueOnError)
+ var dryRun bool
+ fs.BoolVar(&dryRun, "dry-run", false, "print what would be done only")
+ fs.Usage = func() {
+ fmt.Fprintln(os.Stderr, "push usage: gomote push <instance>")
+ fs.PrintDefaults()
+ os.Exit(1)
+ }
+ fs.Parse(args)
+
+ goroot := os.Getenv("GOROOT")
+ if goroot == "" {
+ slurp, err := exec.Command("go", "env", "GOROOT").Output()
+ if err != nil {
+ return fmt.Errorf("failed to get GOROOT from go env: %v", err)
+ }
+ goroot = strings.TrimSpace(string(slurp))
+ if goroot == "" {
+ return errors.New("Failed to get $GOROOT from environment or go env")
+ }
+ }
+ goroot = filepath.Clean(goroot)
+
+ if fs.NArg() != 1 {
+ fs.Usage()
+ }
+ name := fs.Arg(0)
+
+ haveGo14 := false
+ remote := map[string]buildlet.DirEntry{} // keys like "src/make.bash"
+
+ ctx := context.Background()
+ client := gomoteServerClient(ctx)
+ resp, err := client.ListDirectory(ctx, &protos.ListDirectoryRequest{
+ GomoteId: name,
+ Directory: ".",
+ Recursive: true,
+ SkipFiles: []string{
+ // Ignore binary output directories:
+ "go/pkg", "go/bin",
+ // We don't care about the digest of
+ // particular source files for Go 1.4. And
+ // exclude /pkg. This leaves go1.4/bin, which
+ // is enough to know whether we have Go 1.4 or
+ // not.
+ "go1.4/src", "go1.4/pkg",
+ },
+ Digest: true,
+ })
+ if err != nil {
+ return fmt.Errorf("error listing buildlet's existing files: %s", statusFromError(err))
+ }
+ for _, entry := range resp.GetEntries() {
+ if strings.HasPrefix(entry, "go1.4/") {
+ haveGo14 = true
+ continue
+ }
+ if strings.HasPrefix(entry, "go/") {
+ remote[name[len("go/"):]] = buildlet.DirEntry{Line: entry}
+ }
+ }
+ if !haveGo14 {
+ log.Printf("installing go1.4")
+ if dryRun {
+ log.Printf("(Dry-run) Would have pushed go1.4")
+ } else {
+ _, err := client.AddBootstrap(ctx, &protos.AddBootstrapRequest{
+ GomoteId: name,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to add bootstrap version of Go to instance: %s", statusFromError(err))
+ }
+ }
+ }
+
+ // Invoke 'git check-ignore' and use it to query whether paths have been gitignored.
+ // If anything goes wrong at any point, fall back to assuming that nothing is gitignored.
+ var isGitIgnored func(string) bool
+ gci := exec.Command("git", "check-ignore", "--stdin", "-n", "-v", "-z")
+ gci.Env = append(os.Environ(), "GIT_FLUSH=1")
+ gciIn, errIn := gci.StdinPipe()
+ defer gciIn.Close() // allow git process to exit
+ gciOut, errOut := gci.StdoutPipe()
+ errStart := gci.Start()
+ if errIn != nil || errOut != nil || errStart != nil {
+ isGitIgnored = func(string) bool { return false }
+ } else {
+ var failed bool
+ br := bufio.NewReader(gciOut)
+ isGitIgnored = func(path string) bool {
+ if failed {
+ return false
+ }
+ fmt.Fprintf(gciIn, "%s\x00", path)
+ // Response is of form "<source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname> <NULL>"
+ // Read all four and check that the path is correct.
+ // If so, the path is ignored iff the source (reason why ignored) is non-empty.
+ var resp [4][]byte
+ for i := range resp {
+ b, err := br.ReadBytes(0)
+ if err != nil {
+ failed = true
+ return false
+ }
+ resp[i] = b[:len(b)-1] // drop trailing NULL
+ }
+ // Sanity check
+ if string(resp[3]) != path {
+ panic("git check-ignore path did not roundtrip, got " + string(resp[3]) + " sent " + path)
+ }
+ return len(resp[0]) > 0
+ }
+ }
+
+ type fileInfo struct {
+ fi os.FileInfo
+ sha1 string // if regular file
+ }
+ local := map[string]fileInfo{} // keys like "src/make.bash"
+ if err := filepath.Walk(goroot, func(path string, fi os.FileInfo, err error) error {
+ if isEditorBackup(path) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ rel, err := filepath.Rel(goroot, path)
+ if err != nil {
+ return fmt.Errorf("error calculating relative path from %q to %q", goroot, path)
+ }
+ rel = filepath.ToSlash(rel)
+ if rel == "." {
+ return nil
+ }
+ if fi.IsDir() {
+ switch rel {
+ case ".git", "pkg", "bin":
+ return filepath.SkipDir
+ }
+ }
+ inf := fileInfo{fi: fi}
+ if isGitIgnored(path) {
+ if fi.Mode().IsDir() {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ if fi.Mode().IsRegular() {
+ inf.sha1, err = fileSHA1(path)
+ if err != nil {
+ return err
+ }
+ }
+ local[rel] = inf
+ return nil
+ }); err != nil {
+ return fmt.Errorf("error enumerating local GOROOT files: %v", err)
+ }
+
+ var toDel []string
+ for rel := range remote {
+ if rel == "VERSION" {
+ // Don't delete this. It's harmless, and
+ // necessary. Clients can overwrite it if they
+ // want. But if there's no VERSION file there,
+ // make.bash/bat assumes there's a git repo in
+ // place, but there's not only not a git repo
+ // there with gomote, but there's no git tool
+ // available either.
+ continue
+ }
+ // Also don't delete the auto-generated files from cmd/dist.
+ // Otherwise gomote users can't gomote push + gomote run make.bash
+ // and then iteratively:
+ // -- hack locally
+ // -- gomote push
+ // -- gomote run go test -v ...
+ // Because the go test would fail remotely without
+ // these files if they were deleted by gomote push.
+ if isGoToolDistGenerated(rel) {
+ continue
+ }
+ if isGitIgnored(rel) {
+ // Don't delete remote gitignored files; this breaks built toolchains.
+ continue
+ }
+ rel = strings.TrimRight(rel, "/")
+ if rel == "" {
+ continue
+ }
+ if _, ok := local[rel]; !ok {
+ toDel = append(toDel, rel)
+ }
+ }
+ if len(toDel) > 0 {
+ withGo := make([]string, len(toDel)) // with the "go/" prefix
+ for i, v := range toDel {
+ withGo[i] = "go/" + v
+ }
+ sort.Strings(withGo)
+ if dryRun {
+ log.Printf("(Dry-run) Would have deleted remote files: %q", withGo)
+ } else {
+ log.Printf("Deleting remote files: %q", withGo)
+ if _, err := client.RemoveFiles(ctx, &protos.RemoveFilesRequest{
+ GomoteId: name,
+ Paths: withGo,
+ }); err != nil {
+ return fmt.Errorf("failed to delete remote unwanted files: %s", statusFromError(err))
+ }
+ }
+ }
+ var toSend []string
+ notHave := 0
+ const maxNotHavePrint = 5
+ for rel, inf := range local {
+ if isGoToolDistGenerated(rel) || rel == "VERSION.cache" {
+ continue
+ }
+ if !inf.fi.Mode().IsRegular() {
+ if !inf.fi.IsDir() {
+ log.Printf("Ignoring local non-regular, non-directory file %s: %v", rel, inf.fi.Mode())
+ }
+ continue
+ }
+ rem, ok := remote[rel]
+ if !ok {
+ if notHave++; notHave <= maxNotHavePrint {
+ log.Printf("Remote doesn't have %q", rel)
+ }
+ toSend = append(toSend, rel)
+ continue
+ }
+ if rem.Digest() != inf.sha1 {
+ log.Printf("Remote's %s digest is %q; want %q", rel, rem.Digest(), inf.sha1)
+ toSend = append(toSend, rel)
+ }
+ }
+ if notHave > maxNotHavePrint {
+ log.Printf("Remote doesn't have %d files (only showed %d).", notHave, maxNotHavePrint)
+ }
+ if _, hasVersion := remote["VERSION"]; !hasVersion {
+ log.Printf("Remote lacks a VERSION file; sending a fake one")
+ toSend = append(toSend, "VERSION")
+ }
+ if len(toSend) > 0 {
+ sort.Strings(toSend)
+ tgz, err := generateDeltaTgz(goroot, toSend)
+ if err != nil {
+ return err
+ }
+ log.Printf("Uploading %d new/changed files; %d byte .tar.gz", len(toSend), tgz.Len())
+ if dryRun {
+ log.Printf("(Dry-run mode; not doing anything.")
+ return nil
+ }
+ resp, err := client.UploadFile(ctx, &protos.UploadFileRequest{})
+ if err != nil {
+ return fmt.Errorf("unable to request credentials for a file upload: %s", statusFromError(err))
+ }
+ if err := uploadToGCS(ctx, resp.GetFields(), tgz, resp.GetObjectName(), resp.GetUrl()); err != nil {
+ return fmt.Errorf("unable to upload file to GCS: %s", err)
+ }
+ if _, err := client.WriteTGZFromURL(ctx, &protos.WriteTGZFromURLRequest{
+ GomoteId: name,
+ Url: fmt.Sprintf("%s%s", resp.GetUrl(), resp.GetObjectName()),
+ }); err != nil {
+ return fmt.Errorf("failed writing tarball to buildlet: %s", statusFromError(err))
+ }
+ }
+ return nil
+}
+
func isGoToolDistGenerated(path string) bool {
switch path {
case "src/cmd/cgo/zdefaultcc.go",
