dashboard/buildlet: start of the buildlet
This is the basics: untar a tar.gz file to a directory, and execute a
command.
Update golang/go#8639
Update golang/go#8640
Update golang/go#8642
Change-Id: I5917ed8bd0e4c2fdb4b3fab34ca929caca95cc8a
Reviewed-on: https://go-review.googlesource.com/2180
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/README b/README
index 94193cd..59b1504 100644
--- a/README
+++ b/README
@@ -5,6 +5,10 @@
The files in this directory constitute the continuous builder:
app/: an AppEngine server. The code that runs http://build.golang.org/
+buildlet/: HTTP server that runs on a VM and is told what to write to disk
+ and what command to run. This is cross-compiled to all architectures
+ and is the first program run when a builder VM comes up. It then
+ is contacted by the coordinator to do a build.
builder/: gobuilder, a Go continuous build client
coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
builds (using the builder in single-shot mode) in Docker containers.
diff --git a/buildlet/buildlet.go b/buildlet/buildlet.go
new file mode 100644
index 0000000..135e816
--- /dev/null
+++ b/buildlet/buildlet.go
@@ -0,0 +1,252 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The buildlet is an HTTP server that untars content to disk and runs
+// commands it has untarred, streaming their output back over HTTP.
+// It is part of Go's continuous build system.
+//
+// This program intentionally allows remote code execution, and
+// provides no security of its own. It is assumed that any user uses
+// it with an appropriately-configured firewall between their VM
+// instances.
+package main // import "golang.org/x/tools/dashboard/buildlet"
+
+/* Notes:
+
+https://go.googlesource.com/go/+archive/3b76b017cabb.tar.gz
+curl -X PUT --data-binary "@go-3b76b017cabb.tar.gz" http://127.0.0.1:5937/writetgz
+
+curl -d "cmd=src/make.bash" http://127.0.0.1:5937/exec
+
+*/
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "sync"
+ "time"
+)
+
+var (
+ scratchDir = flag.String("scratchdir", "", "Temporary directory to use. The contents of this directory may be deleted at any time. If empty, TempDir is used to create one.")
+ listenAddr = flag.String("listen", defaultListenAddr(), "address to listen on. Warning: this service is inherently insecure and offers no protection of its own. Do not expose this port to the world.")
+)
+
+func defaultListenAddr() string {
+ if OnGCE() {
+ // In production, default to
+ return ":80"
+ }
+ return "localhost:5936"
+}
+
+func main() {
+ flag.Parse()
+ if !OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") {
+ log.Printf("** WARNING *** This server is unsafe and offers no security. Be careful.")
+ }
+ if *scratchDir == "" {
+ dir, err := ioutil.TempDir("", "buildlet-scatch")
+ if err != nil {
+ log.Fatalf("error creating scratchdir with ioutil.TempDir: %v", err)
+ }
+ *scratchDir = dir
+ }
+ if _, err := os.Lstat(*scratchDir); err != nil {
+ log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
+ }
+ http.HandleFunc("/writetgz", handleWriteTGZ)
+ http.HandleFunc("/exec", handleExec)
+ http.HandleFunc("/", handleRoot)
+ // TODO: removeall
+ log.Printf("Listening on %s ...", *listenAddr)
+ log.Fatalf("ListenAndServe: %v", http.ListenAndServe(*listenAddr, nil))
+}
+
+func handleRoot(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, "buildlet running on %s-%s", runtime.GOOS, runtime.GOARCH)
+}
+
+func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
+ if r.Method != "PUT" {
+ http.Error(w, "requires PUT method", http.StatusBadRequest)
+ return
+ }
+ err := untar(r.Body, *scratchDir)
+ if err != nil {
+ status := http.StatusInternalServerError
+ if he, ok := err.(httpStatuser); ok {
+ status = he.httpStatus()
+ }
+ http.Error(w, err.Error(), status)
+ return
+ }
+ io.WriteString(w, "OK")
+}
+
+// untar reads the gzip-compressed tar file from r and writes it into dir.
+func untar(r io.Reader, dir string) error {
+ zr, err := gzip.NewReader(r)
+ if err != nil {
+ return badRequest("requires gzip-compressed body: " + err.Error())
+ }
+ tr := tar.NewReader(zr)
+ for {
+ f, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ log.Printf("tar reading error: %v", err)
+ return badRequest("tar error: " + err.Error())
+ }
+ if !validRelPath(f.Name) {
+ return badRequest(fmt.Sprintf("tar file contained invalid name %q", f.Name))
+ }
+ rel := filepath.FromSlash(f.Name)
+ abs := filepath.Join(dir, rel)
+
+ fi := f.FileInfo()
+ mode := fi.Mode()
+ switch {
+ case mode.IsRegular():
+ // Make the directory. This is redundant because it should
+ // already be made by a directory entry in the tar
+ // beforehand. Thus, don't check for errors; the next
+ // write will fail with the same error.
+ os.MkdirAll(filepath.Dir(abs), 0755)
+ wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+ if err != nil {
+ return err
+ }
+ n, err := io.Copy(wf, tr)
+ if closeErr := wf.Close(); closeErr != nil && err == nil {
+ err = closeErr
+ }
+ if err != nil {
+ return fmt.Errorf("error writing to %s: %v", abs, err)
+ }
+ if n != f.Size {
+ return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
+ }
+ log.Printf("wrote %s", abs)
+ case mode.IsDir():
+ if err := os.MkdirAll(abs, 0755); err != nil {
+ return err
+ }
+ default:
+ return badRequest(fmt.Sprintf("tar file entry %s contained unsupported file type %v", f.Name, mode))
+ }
+ }
+ return nil
+}
+
+func handleExec(w http.ResponseWriter, r *http.Request) {
+ if r.Method != "POST" {
+ http.Error(w, "requires POST method", http.StatusBadRequest)
+ return
+ }
+ cmdPath := r.FormValue("cmd") // required
+ if !validRelPath(cmdPath) {
+ http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
+ return
+ }
+ absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
+ cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
+ cmd.Dir = filepath.Dir(absCmd)
+ cmdOutput := &flushWriter{w: w}
+ cmd.Stdout = cmdOutput
+ cmd.Stderr = cmdOutput
+ err := cmd.Run()
+ log.Printf("Run = %v", err)
+ // TODO: put the exit status in the HTTP trailer,
+ // once https://golang.org/issue/7759 is fixed.
+}
+
+// flushWriter is an io.Writer wrapper that writes to w and
+// Flushes the output immediately, if w is an http.Flusher.
+type flushWriter struct {
+ mu sync.Mutex
+ w http.ResponseWriter
+}
+
+func (hw *flushWriter) Write(p []byte) (n int, err error) {
+ hw.mu.Lock()
+ defer hw.mu.Unlock()
+ n, err = hw.w.Write(p)
+ if f, ok := hw.w.(http.Flusher); ok {
+ f.Flush()
+ }
+ return
+}
+
+func validRelPath(p string) bool {
+ if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+ return false
+ }
+ return true
+}
+
+type httpStatuser interface {
+ error
+ httpStatus() int
+}
+
+type httpError struct {
+ statusCode int
+ msg string
+}
+
+func (he httpError) Error() string { return he.msg }
+func (he httpError) httpStatus() int { return he.statusCode }
+
+func badRequest(msg string) error {
+ return httpError{http.StatusBadRequest, msg}
+}
+
+// metaClient to fetch GCE metadata values.
+var metaClient = &http.Client{
+ Transport: &http.Transport{
+ Dial: (&net.Dialer{
+ Timeout: 750 * time.Millisecond,
+ KeepAlive: 30 * time.Second,
+ }).Dial,
+ ResponseHeaderTimeout: 750 * time.Millisecond,
+ },
+}
+
+var onGCE struct {
+ sync.Mutex
+ set bool
+ v bool
+}
+
+// OnGCE reports whether this process is running on Google Compute Engine.
+func OnGCE() bool {
+ defer onGCE.Unlock()
+ onGCE.Lock()
+ if onGCE.set {
+ return onGCE.v
+ }
+ onGCE.set = true
+
+ res, err := metaClient.Get("http://metadata.google.internal")
+ if err != nil {
+ return false
+ }
+ onGCE.v = res.Header.Get("Metadata-Flavor") == "Google"
+ return onGCE.v
+}
