buildlet: make exec fail with a remote error if no headers after 5 seconds

The reverse buildlets' RoundTrip are hanging, which is its own problem,
but this calling code should be robust and time out anyway.

Change-Id: Id9e3e1d9feb6ffa58cc0995d0623bd90845bb9d6
Reviewed-on: https://go-review.googlesource.com/10847
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/buildlet/buildletclient.go b/buildlet/buildletclient.go
index a840744..53b3445 100644
--- a/buildlet/buildletclient.go
+++ b/buildlet/buildletclient.go
@@ -114,6 +114,37 @@
 	return c.httpClient.Do(req)
 }
 
+var errHeaderTimeout = errors.New("timeout waiting for headers")
+
+// doHeaderTimeout calls c.do(req) and returns its results, or
+// errHeaderTimeout if max elapses first.
+func (c *Client) doHeaderTimeout(req *http.Request, max time.Duration) (res *http.Response, err error) {
+	type resErr struct {
+		res *http.Response
+		err error
+	}
+	resErrc := make(chan resErr, 1)
+	go func() {
+		res, err := c.do(req)
+		resErrc <- resErr{res, err}
+	}()
+
+	timer := time.NewTimer(max)
+	defer timer.Stop()
+
+	select {
+	case re := <-resErrc:
+		return re.res, re.err
+	case <-timer.C:
+		go func() {
+			if re := <-resErrc; re.res != nil {
+				res.Body.Close()
+			}
+		}()
+		return nil, errHeaderTimeout
+	}
+}
+
 // doOK sends the request and expects a 200 OK response.
 func (c *Client) doOK(req *http.Request) error {
 	res, err := c.do(req)
@@ -264,7 +295,14 @@
 		return nil, err
 	}
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	res, err := c.do(req)
+
+	// The first thing the buildlet's exec handler does is flush the headers, so
+	// 5 seconds should be plenty of time, regardless of where on the planet
+	// (Atlanta, Paris, etc) the reverse buildlet is:
+	res, err := c.doHeaderTimeout(req, 5*time.Second)
+	if err == errHeaderTimeout {
+		return nil, errors.New("buildlet: timeout waiting for exec header response")
+	}
 	if err != nil {
 		return nil, err
 	}