cmd/buildlet/stage0: add untar mode
The new untar package will be shared by the buildlet later.
Change-Id: I7a33bf46e5cb6b7e5abd59613bb943039c84aa08
Reviewed-on: https://go-review.googlesource.com/41131
Reviewed-by: Jeff Johnson <jrjohnson@google.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/cmd/buildlet/stage0/stage0.go b/cmd/buildlet/stage0/stage0.go
index 05ac24a..2256c5e 100644
--- a/cmd/buildlet/stage0/stage0.go
+++ b/cmd/buildlet/stage0/stage0.go
@@ -28,6 +28,7 @@
"cloud.google.com/go/compute/metadata"
"golang.org/x/build/internal/httpdl"
+ "golang.org/x/build/internal/untar"
)
// This lets us be lazy and put the stage0 start-up in rc.local where
@@ -44,9 +45,20 @@
scalewayMeta scalewayMetadata
)
+// untar helper, for the Windows image prep script.
+var (
+ untarFile = flag.String("untar-file", "", "if non-empty, tar.gz to untar to --untar-dest-dir")
+ untarDestDir = flag.String("untar-dest-dir", "", "destination directory to untar --untar-file to")
+)
+
func main() {
flag.Parse()
+ if *untarFile != "" {
+ untarMode()
+ return
+ }
+
switch osArch {
case "linux/arm":
if os.Getenv("GO_BUILDER_ENV") == "linux-arm-arm5spacemonkey" {
@@ -441,3 +453,23 @@
}
return true
}
+
+func untarMode() {
+ if *untarDestDir == "" {
+ log.Fatal("--untar-dest-dir must not be empty")
+ }
+ if fi, err := os.Stat(*untarDestDir); err != nil || !fi.IsDir() {
+ if err != nil {
+ log.Fatalf("--untar-dest-dir %q: %v", *untarDestDir, err)
+ }
+ log.Fatalf("--untar-dest-dir %q not a directory.", *untarDestDir)
+ }
+ f, err := os.Open(*untarFile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer f.Close()
+ if err := untar.Untar(f, *untarDestDir); err != nil {
+ log.Fatalf("Untarring %q to %q: %v", *untarFile, *untarDestDir)
+ }
+}
diff --git a/internal/untar/untar.go b/internal/untar/untar.go
new file mode 100644
index 0000000..1083395
--- /dev/null
+++ b/internal/untar/untar.go
@@ -0,0 +1,141 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package untar untars a tarball to disk.
+package untar
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// TODO(bradfitz): this was copied from x/build/cmd/buildlet/buildlet.go
+// but there were some buildlet-specific bits in there, so the code is
+// forked for now. Unfork and add some opts arguments here, so the
+// buildlet can use this code somehow.
+
+// Untar reads the gzip-compressed tar file from r and writes it into dir.
+func Untar(r io.Reader, dir string) error {
+ return untar(r, dir)
+}
+
+func untar(r io.Reader, dir string) (err error) {
+ t0 := time.Now()
+ nFiles := 0
+ madeDir := map[string]bool{}
+ defer func() {
+ td := time.Since(t0)
+ if err == nil {
+ log.Printf("extracted tarball into %s: %d files, %d dirs (%v)", dir, nFiles, len(madeDir), td)
+ } else {
+ log.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
+ }
+ }()
+ zr, err := gzip.NewReader(r)
+ if err != nil {
+ return fmt.Errorf("requires gzip-compressed body: %v", err)
+ }
+ tr := tar.NewReader(zr)
+ loggedChtimesError := false
+ for {
+ f, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ log.Printf("tar reading error: %v", err)
+ return fmt.Errorf("tar error: %v", err)
+ }
+ if !validRelPath(f.Name) {
+ return fmt.Errorf("tar contained invalid name error %q", f.Name)
+ }
+ rel := filepath.FromSlash(f.Name)
+ abs := filepath.Join(dir, rel)
+
+ fi := f.FileInfo()
+ mode := fi.Mode()
+ switch {
+ case mode.IsRegular():
+ // Make the directory. This is redundant because it should
+ // already be made by a directory entry in the tar
+ // beforehand. Thus, don't check for errors; the next
+ // write will fail with the same error.
+ dir := filepath.Dir(abs)
+ if !madeDir[dir] {
+ if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil {
+ return err
+ }
+ madeDir[dir] = true
+ }
+ wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+ if err != nil {
+ return err
+ }
+ n, err := io.Copy(wf, tr)
+ if closeErr := wf.Close(); closeErr != nil && err == nil {
+ err = closeErr
+ }
+ if err != nil {
+ return fmt.Errorf("error writing to %s: %v", abs, err)
+ }
+ if n != f.Size {
+ return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
+ }
+ modTime := f.ModTime
+ if modTime.After(t0) {
+ // Clamp modtimes at system time. See
+ // golang.org/issue/19062 when clock on
+ // buildlet was behind the gitmirror server
+ // doing the git-archive.
+ modTime = t0
+ }
+ if !modTime.IsZero() {
+ if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError {
+ // benign error. Gerrit doesn't even set the
+ // modtime in these, and we don't end up relying
+ // on it anywhere (the gomote push command relies
+ // on digests only), so this is a little pointless
+ // for now.
+ log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err)
+ loggedChtimesError = true // once is enough
+ }
+ }
+ nFiles++
+ case mode.IsDir():
+ if err := os.MkdirAll(abs, 0755); err != nil {
+ return err
+ }
+ madeDir[abs] = true
+ default:
+ return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode)
+ }
+ }
+ return nil
+}
+
+func validRelativeDir(dir string) bool {
+ if strings.Contains(dir, `\`) || path.IsAbs(dir) {
+ return false
+ }
+ dir = path.Clean(dir)
+ if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." {
+ return false
+ }
+ return true
+}
+
+func validRelPath(p string) bool {
+ if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+ return false
+ }
+ return true
+}
