all: improved monitoring of buildlet pods after creation

* Replaced cancel with context.Context
* StartPod can be canceled
* Wait for buildlet to come online, but fail fast if pod fails first
* Support timeout waiting for pod to leave pending phase
* Use Kubernetes watch API (long poll)

Updates golang/go#12546

Change-Id: I792a3b8fed615362a0290feee7de0c2cefe43c0e
Reviewed-on: https://go-review.googlesource.com/15285
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/cmd/coordinator/remote.go b/cmd/coordinator/remote.go
index 41dd59d..a79aa16 100644
--- a/cmd/coordinator/remote.go
+++ b/cmd/coordinator/remote.go
@@ -21,6 +21,7 @@
 
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/dashboard"
+	"golang.org/x/net/context"
 )
 
 var (
@@ -106,12 +107,13 @@
 	if cn, ok := w.(http.CloseNotifier); ok {
 		closeNotify = cn.CloseNotify()
 	}
-	cancel := make(chan struct{})
+
+	ctx, cancel := context.WithCancel(context.Background())
 
 	resc := make(chan *buildlet.Client)
 	errc := make(chan error)
 	go func() {
-		bc, err := pool.GetBuildlet(cancel, typ, rev, eventTimeLoggerFunc(func(event string, optText ...string) {
+		bc, err := pool.GetBuildlet(ctx, typ, rev, eventTimeLoggerFunc(func(event string, optText ...string) {
 			var extra string
 			if len(optText) > 0 {
 				extra = " " + optText[0]
@@ -152,7 +154,7 @@
 			return
 		case <-closeNotify:
 			log.Printf("client went away during buildlet create request")
-			close(cancel)
+			cancel()
 			closeNotify = nil // unnecessary, but habit.
 		}
 	}