cmd/coordinator, cmd/buildlet, cmd/gomote: add SSH support
This adds an SSH server to farmer.golang.org on port 2222 that proxies
SSH connections to users' gomote-created buildlet instances.
For example:
$ gomote create openbsd-amd64-60
user-bradfitz-openbsd-amd64-60-1
$ gomote ssh user-bradfitz-openbsd-amd64-60-1
Warning: Permanently added '[localhost]:33351' (ECDSA) to the list of known hosts.
OpenBSD 6.0 (GENERIC.MP) golang/go#2319: Tue Jul 26 13:00:43 MDT 2016
Welcome to OpenBSD: The proactively secure Unix-like operating system.
Please use the sendbug(1) utility to report bugs in the system.
Before reporting a bug, please try to reproduce it with the latest
version of the code. With bug reports, please try to ensure that
enough information to reproduce the problem is enclosed, and if a
known fix for it exists, include that as well.
$
As before, if the coordinator process is restarted (or crashes, is
evicted, etc), all gomote instances die.
Not yet supported:
* scp (help wanted)
* not all host types are configured. most are. some will need slight
config tweaks to the Docker image (e.g. adding openssh-server)
Supports currently:
* linux-amd64 (host type shared by 386, nacl)
* linux-arm
* linux-arm64
* darwin
* freebsd
* openbsd
* plan9-386
* windows
Implementation details:
* the ssh server process listens on port 2222 in the coordinator
(farmer.golang.org), which is behind a GKE TCP load balancer.
* the ssh server library is github.com/gliderlabs/ssh
* authentication is done via Github users' public keys. It's assumed
that gomote user == github user. But there's a mapping in the code
for known exceptions.
* we can't give out access to this too widely. too many things are
accessible from within the host environment if you look in the right
places. Details omitted. But the Go team and other trusted gomote
users can use this.
* the buildlet binary has a new /connect-ssh handler that acts like a
CONNECT request but instead of taking an explicit host:port, just
says "give me your machine's SSH connection". The buildlet can also
start sshd if needed for the environment. The /connect-ssh handler
also installs the coordinator's public key.
* a new buildlet client library method "ConnectSSH" hits the /connect-ssh
handler and returns a net.Conn.
* the coordinator's ssh.Handler is just running the OpenSSH ssh client.
* because the OpenSSH ssh child process can't connect to a net.Conn,
an emphemeral localhost port is created on the coordinator to proxy
between the ssh client and the net.Conn returned by ConnectSSH.
* The /connect-ssh handler requires http.Hijacker, which requires
fully compliant net.Conn implementations as of Go 1.8. So I needed
to flesh out revdial too, testing it with the
golang.org/x/net/nettest package.
* plan9 doesn't have an ssh server, so we use 0intro's new conterm
program (drawterm without GUI support) to connect to plan9 from the
coordinator ssh proxy instead of using the OpenSSH ssh client
binary.
* windows doesn't have an ssh server, so we enable the telnet service
and the coordinator ssh proxy uses telnet instead on the backend
on the private network. (There is a Windows ssh server but only in
new versions.)
Happy debugging over ssh!
Fixes golang/go#19956
Change-Id: I80a62064c5f85af1f195f980c862ba29af4015f0
Reviewed-on: https://go-review.googlesource.com/50750
Reviewed-by: Herbie Ong <herbie@google.com>
Reviewed-by: Jessie Frazelle <me@jessfraz.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/buildenv/envs.go b/buildenv/envs.go
index 611106c..4dde252 100644
--- a/buildenv/envs.go
+++ b/buildenv/envs.go
@@ -240,12 +240,14 @@
var (
registeredFlags bool
stagingFlag bool
+ localDevFlag bool
)
// RegisterFlags registers the "staging" flag. It is required if FromFlags is used.
func RegisterFlags() {
if !registeredFlags {
flag.BoolVar(&stagingFlag, "staging", false, "use the staging build coordinator and buildlets")
+ flag.BoolVar(&localDevFlag, "localdev", false, "use the localhost in-development coordinator")
registeredFlags = true
}
}
@@ -256,6 +258,9 @@
if !registeredFlags {
panic("FromFlags called without RegisterFlags")
}
+ if localDevFlag {
+ return Development
+ }
if stagingFlag {
return Staging
}
diff --git a/buildlet/buildletclient.go b/buildlet/buildletclient.go
index 527969c..0e03d83 100644
--- a/buildlet/buildletclient.go
+++ b/buildlet/buildletclient.go
@@ -116,6 +116,12 @@
c.httpClient = httpClient
}
+// SetDialer sets the function that creates a new connection to the buildlet.
+// By default, net.Dial is used.
+func (c *Client) SetDialer(dialer func() (net.Conn, error)) {
+ c.dialer = dialer
+}
+
// defaultDialer returns the net/http package's default Dial function.
// Notably, this sets TCP keep-alive values, so when we kill VMs
// (whose TCP stacks stop replying, forever), we don't leak file
@@ -132,10 +138,11 @@
ipPort string // required, unless remoteBuildlet+baseURL is set
tls KeyPair
httpClient *http.Client
- baseURL string // optional baseURL (used by remote buildlets)
- authUser string // defaults to "gomote", if password is non-empty
- password string // basic auth password or empty for none
- remoteBuildlet string // non-empty if for remote buildlets
+ dialer func() (net.Conn, error) // nil means to use net.Dial
+ baseURL string // optional baseURL (used by remote buildlets)
+ authUser string // defaults to "gomote", if password is non-empty
+ password string // basic auth password or empty for none
+ remoteBuildlet string // non-empty if for remote buildlets
closeFuncs []func() // optional extra code to run on close
releaseMode bool
@@ -745,6 +752,53 @@
return sc.Err()
}
+func (c *Client) getDialer() func() (net.Conn, error) {
+ if c.dialer != nil {
+ return c.dialer
+ }
+ return c.dialWithNetDial
+}
+
+func (c *Client) dialWithNetDial() (net.Conn, error) {
+ // TODO: contexts? the tedious part will be adding it to
+ // revdial.Dial. For now just do a 5 second timeout. Probably
+ // fine. This is currently only used for ssh connections.
+ d := net.Dialer{Timeout: 5 * time.Second}
+ return d.Dial("tcp", c.ipPort)
+}
+
+// ConnectSSH opens an SSH connection to the buildlet for the given username.
+// The authorizedPubKey must be a line from an ~/.ssh/authorized_keys file
+// and correspond to the private key to be used to communicate over the net.Conn.
+func (c *Client) ConnectSSH(user, authorizedPubKey string) (net.Conn, error) {
+ conn, err := c.getDialer()()
+ if err != nil {
+ return nil, fmt.Errorf("error dialing HTTP connection before SSH upgrade: %v", err)
+ }
+ req, err := http.NewRequest("POST", "/connect-ssh", nil)
+ if err != nil {
+ conn.Close()
+ return nil, err
+ }
+ req.Header.Add("X-Go-Ssh-User", user)
+ req.Header.Add("X-Go-Authorized-Key", authorizedPubKey)
+ if err := req.Write(conn); err != nil {
+ conn.Close()
+ return nil, fmt.Errorf("writing /connect-ssh HTTP request failed: %v", err)
+ }
+ bufr := bufio.NewReader(conn)
+ res, err := http.ReadResponse(bufr, req)
+ if err != nil {
+ conn.Close()
+ return nil, fmt.Errorf("reading /connect-ssh response: %v", err)
+ }
+ if res.StatusCode != http.StatusSwitchingProtocols {
+ slurp, _ := ioutil.ReadAll(res.Body)
+ return nil, fmt.Errorf("unexpected /connect-ssh response: %v, %s", res.Status, slurp)
+ }
+ return conn, nil
+}
+
func condRun(fn func()) {
if fn != nil {
fn()
diff --git a/buildlet/remote.go b/buildlet/remote.go
index e233a61..ae05892 100644
--- a/buildlet/remote.go
+++ b/buildlet/remote.go
@@ -229,9 +229,13 @@
return nil, errors.New("RegisterFlags not called")
}
inst := build.ProdCoordinator
- if buildenv.FromFlags() == buildenv.Staging {
+ env := buildenv.FromFlags()
+ if env == buildenv.Staging {
inst = build.StagingCoordinator
+ } else if env == buildenv.Development {
+ inst = "localhost:8119"
}
+
if gomoteUserFlag == "" {
return nil, errors.New("user flag must be specified")
}
diff --git a/cmd/buildlet/buildlet.go b/cmd/buildlet/buildlet.go
index c8b58b8..3095c12 100644
--- a/cmd/buildlet/buildlet.go
+++ b/cmd/buildlet/buildlet.go
@@ -38,6 +38,7 @@
"runtime"
"strconv"
"strings"
+ "sync"
"time"
"cloud.google.com/go/compute/metadata"
@@ -69,7 +70,8 @@
// 7: version bumps while debugging revdial hang (Issue 12816)
// 8: mac screensaver disabled
// 11: move from self-signed cert to LetsEncrypt (Issue 16442)
-const buildletVersion = 11
+// 15: ssh support
+const buildletVersion = 15
func defaultListenAddr() string {
if runtime.GOOS == "darwin" {
@@ -202,6 +204,7 @@
http.Handle("/workdir", requireAuth(handleWorkDir))
http.Handle("/status", requireAuth(handleStatus))
http.Handle("/ls", requireAuth(handleLs))
+ http.Handle("/connect-ssh", requireAuth(handleConnectSSH))
if !isReverse {
listenForCoordinator()
@@ -527,10 +530,7 @@
})
if err != nil {
log.Printf("Walk error: %v", err)
- // Decent way to signal failure to the caller, since it'll break
- // the chunked response, rather than have a valid EOF.
- conn, _, _ := w.(http.Hijacker).Hijack()
- conn.Close()
+ panic(http.ErrAbortHandler)
}
tw.Close()
zw.Close()
@@ -1230,6 +1230,147 @@
}
}
+func handleConnectSSH(w http.ResponseWriter, r *http.Request) {
+ if r.Method != "POST" {
+ http.Error(w, "requires POST method", http.StatusBadRequest)
+ return
+ }
+ if r.ContentLength != 0 {
+ http.Error(w, "requires zero Content-Length", http.StatusBadRequest)
+ return
+ }
+ sshUser := r.Header.Get("X-Go-Ssh-User")
+ authKey := r.Header.Get("X-Go-Authorized-Key")
+ if sshUser != "" && authKey != "" {
+ if err := appendSSHAuthorizedKey(sshUser, authKey); err != nil {
+ http.Error(w, "adding ssh authorized key: "+err.Error(), http.StatusBadRequest)
+ return
+ }
+ }
+
+ sshConn, err := net.Dial("tcp", "localhost:22")
+ if err != nil {
+ sshServerOnce.Do(startSSHServer)
+ sshConn, err = net.Dial("tcp", "localhost:22")
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ }
+ defer sshConn.Close()
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ log.Printf("conn can't hijack for ssh proxy; HTTP/2 enabled by default?")
+ http.Error(w, "conn can't hijack", http.StatusInternalServerError)
+ return
+ }
+ conn, _, err := hj.Hijack()
+ if err != nil {
+ log.Printf("ssh hijack error: %v", err)
+ http.Error(w, "ssh hijack error: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer conn.Close()
+ fmt.Fprintf(conn, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: ssh\r\nConnection: Upgrade\r\n\r\n")
+ errc := make(chan error, 1)
+ go func() {
+ _, err := io.Copy(sshConn, conn)
+ errc <- err
+ }()
+ go func() {
+ _, err := io.Copy(conn, sshConn)
+ errc <- err
+ }()
+ <-errc
+}
+
+var sshServerOnce sync.Once
+
+func startSSHServer() {
+ if inLinuxContainer() {
+ startSSHServerLinux()
+ return
+ }
+
+ log.Printf("start ssh server: don't know how to start SSH server on this host type")
+}
+
+// inLinuxContainer reports whether it looks like we're on Linux running inside a container.
+func inLinuxContainer() bool {
+ if runtime.GOOS != "linux" {
+ return false
+ }
+ if numProcs() >= 4 {
+ // There should 1 process running (this buildlet
+ // binary) if we're in Docker. Maybe 2 if something
+ // else is happening. But if there are 4 or more,
+ // we'll be paranoid and assuming we're running on a
+ // user or host system and don't want to start an ssh
+ // server.
+ return false
+ }
+ // TODO: use a more explicit env variable or on-disk signal
+ // that we're in a Go buildlet Docker image. But for now, this
+ // seems to be consistently true:
+ fi, err := os.Stat("/usr/local/bin/stage0")
+ return err == nil && fi.Mode().IsRegular()
+}
+
+func startSSHServerLinux() {
+ log.Printf("start ssh server for linux")
+
+ // First, create the privsep directory, otherwise we get a successful cmd.Start,
+ // but this error message and then an exit:
+ // Missing privilege separation directory: /var/run/sshd
+ if err := os.MkdirAll("/var/run/sshd", 0700); err != nil {
+ log.Printf("creating /var/run/sshd: %v", err)
+ return
+ }
+
+ // The scaleway Docker images don't have ssh host keys in
+ // their image, at least as of 2017-07-23. So make them first.
+ // These are the types sshd -D complains about currently.
+ if runtime.GOARCH == "arm" {
+ for _, keyType := range []string{"rsa", "dsa", "ed25519", "ecdsa"} {
+ file := "/etc/ssh/ssh_host_" + keyType + "_key"
+ if _, err := os.Stat(file); err == nil {
+ continue
+ }
+ out, err := exec.Command("/usr/bin/ssh-keygen", "-f", file, "-N", "", "-t", keyType).CombinedOutput()
+ log.Printf("ssh-keygen of type %s: err=%v, %s\n", err, out)
+ }
+ }
+
+ cmd := exec.Command("/usr/sbin/sshd", "-D")
+ err := cmd.Start()
+ if err != nil {
+ log.Printf("starting sshd: %v", err)
+ return
+ }
+ log.Printf("sshd started.")
+ for i := 0; i < 40; i++ {
+ time.Sleep(10 * time.Millisecond * time.Duration(i+1))
+ c, err := net.Dial("tcp", "localhost:22")
+ if err == nil {
+ c.Close()
+ log.Printf("sshd connected.")
+ return
+ }
+ }
+ log.Printf("timeout waiting for sshd to come up")
+}
+
+func numProcs() int {
+ n := 0
+ fis, _ := ioutil.ReadDir("/proc")
+ for _, fi := range fis {
+ if _, err := strconv.Atoi(fi.Name()); err == nil {
+ n++
+ }
+ }
+ return n
+}
+
func fileSHA1(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
@@ -1305,7 +1446,7 @@
func requireTrailerSupport() {
// Depend on a symbol that was added after HTTP Trailer support was
- // implemented (4b96409 Dec 29 2014)j so that this function will fail
+ // implemented (4b96409 Dec 29 2014) so that this function will fail
// to compile without Trailer support.
// bufio.Reader.Discard was added by ee2ecc4 Jan 7 2015.
var r bufio.Reader
@@ -1394,3 +1535,53 @@
}
log.Printf("Remounted / with async,noatime.")
}
+
+func appendSSHAuthorizedKey(sshUser, authKey string) error {
+ var homeRoot string
+ switch runtime.GOOS {
+ case "darwin":
+ homeRoot = "/Users"
+ case "windows", "plan9":
+ return fmt.Errorf("ssh not supported on %v", runtime.GOOS)
+ default:
+ homeRoot = "/home"
+ if runtime.GOOS == "freebsd" {
+ if fi, err := os.Stat("/usr/home/" + sshUser); err == nil && fi.IsDir() {
+ homeRoot = "/usr/home"
+ }
+ }
+ if sshUser == "root" {
+ homeRoot = "/"
+ }
+ }
+ sshDir := filepath.Join(homeRoot, sshUser, ".ssh")
+ if err := os.MkdirAll(sshDir, 0700); err != nil {
+ return err
+ }
+ if err := os.Chmod(sshDir, 0700); err != nil {
+ return err
+ }
+ authFile := filepath.Join(sshDir, "authorized_keys")
+ exist, err := ioutil.ReadFile(authFile)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ if strings.Contains(string(exist), authKey) {
+ return nil
+ }
+ f, err := os.OpenFile(authFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+ if err != nil {
+ return err
+ }
+ if _, err := fmt.Fprintf(f, "%s\n", authKey); err != nil {
+ f.Close()
+ return err
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ if runtime.GOOS == "freebsd" {
+ exec.Command("/usr/sbin/chown", "-R", sshUser, sshDir).Run()
+ }
+ return nil
+}
diff --git a/cmd/coordinator/Dockerfile b/cmd/coordinator/Dockerfile
index 4e92885..b92b76b 100644
--- a/cmd/coordinator/Dockerfile
+++ b/cmd/coordinator/Dockerfile
@@ -1,9 +1,29 @@
# Copyright 2017 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
-FROM scratch
+FROM debian:jessie
LABEL maintainer "golang-dev@googlegroups.com"
-COPY ca-certificates.crt /etc/ssl/certs/
+# openssh client is for the gomote ssh proxy client.
+# telnet is for the gomote ssh proxy to windows. (no ssh server there)
+# git-core, make, gcc, libc6-dev, and libx11-dev are to build 0intro/conterm,
+# used to connect to plan9 instances.
+RUN apt-get update && apt-get install -y \
+ --no-install-recommends \
+ ca-certificates \
+ openssh-client \
+ telnet \
+ git-core make gcc libc6-dev libx11-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# drawterm connects to plan9 instances like:
+# echo glenda123 | ./drawterm -a <addr> -c <addr> -u glenda -k user=glenda
+# Where <addr> is the IP address of the Plan 9 instance on GCE,
+# "glenda" is the username and "glenda123" is the password.
+RUN git clone https://github.com/0intro/conterm /tmp/conterm && \
+ cd /tmp/conterm && \
+ CONF=unix make && mv /tmp/conterm/drawterm /usr/local/bin && \
+ rm -rf /tmp/conterm
+
COPY coordinator /
ENTRYPOINT ["/coordinator"]
diff --git a/cmd/coordinator/Dockerfile.0 b/cmd/coordinator/Dockerfile.0
index 2ae6a24..aec44be 100644
--- a/cmd/coordinator/Dockerfile.0
+++ b/cmd/coordinator/Dockerfile.0
@@ -12,6 +12,16 @@
RUN go get -d cloud.google.com/go/compute/metadata `#and 10 other pkgs` &&\
(cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+# Repo github.com/anmitsu/go-shlex at 648efa6 (2016-10-02)
+ENV REV=648efa622239a2f6ff949fed78ee37b48d499ba4
+RUN go get -d github.com/anmitsu/go-shlex &&\
+ (cd /go/src/github.com/anmitsu/go-shlex && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Repo github.com/gliderlabs/ssh at f892d8d (2017-07-20)
+ENV REV=f892d8d851ee02773306439f70af0843395eb9c4
+RUN go get -d github.com/gliderlabs/ssh &&\
+ (cd /go/src/github.com/gliderlabs/ssh && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
# Repo github.com/golang/protobuf at 0a4f71a (2017-07-11)
ENV REV=0a4f71a498b7c4812f64969510bcb4eca251e33a
RUN go get -d github.com/golang/protobuf/proto `#and 9 other pkgs` &&\
@@ -22,14 +32,19 @@
RUN go get -d github.com/googleapis/gax-go &&\
(cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+# Repo github.com/kr/pty at 2c10821 (2017-03-07)
+ENV REV=2c10821df3c3cf905230d078702dfbe9404c9b23
+RUN go get -d github.com/kr/pty &&\
+ (cd /go/src/github.com/kr/pty && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
# Repo go4.org at 034d17a (2017-05-25)
ENV REV=034d17a462f7b2dcd1a4a73553ec5357ff6e6c6e
RUN go get -d go4.org/syncutil &&\
(cd /go/src/go4.org && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/crypto at 94c6142 (2017-07-20)
-ENV REV=94c6142ae57b8dc154f6e1813c921a6c85f505cd
-RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\
+# Repo golang.org/x/crypto at 6914964 (2017-07-20)
+ENV REV=6914964337150723782436d56b3f21610a74ce7b
+RUN go get -d golang.org/x/crypto/acme `#and 6 other pkgs` &&\
(cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/net at ab54850 (2017-07-21)
@@ -67,8 +82,8 @@
RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 10 other pkgs` &&\
(cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo google.golang.org/grpc at 0c41876 (2017-07-21)
-ENV REV=0c41876308d45bc82e587965971e28be659a1aca
+# Repo google.golang.org/grpc at 6495e8d (2017-07-21)
+ENV REV=6495e8dfeb47dc61ff4c392ba6369fa3f15cdc1b
RUN go get -d google.golang.org/grpc `#and 15 other pkgs` &&\
(cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
@@ -93,6 +108,8 @@
cloud.google.com/go/internal/version \
cloud.google.com/go/monitoring/apiv3 \
cloud.google.com/go/storage \
+ github.com/anmitsu/go-shlex \
+ github.com/gliderlabs/ssh \
github.com/golang/protobuf/proto \
github.com/golang/protobuf/protoc-gen-go/descriptor \
github.com/golang/protobuf/ptypes \
@@ -103,9 +120,14 @@
github.com/golang/protobuf/ptypes/timestamp \
github.com/golang/protobuf/ptypes/wrappers \
github.com/googleapis/gax-go \
+ github.com/kr/pty \
go4.org/syncutil \
golang.org/x/crypto/acme \
golang.org/x/crypto/acme/autocert \
+ golang.org/x/crypto/curve25519 \
+ golang.org/x/crypto/ed25519 \
+ golang.org/x/crypto/ed25519/internal/edwards25519 \
+ golang.org/x/crypto/ssh \
golang.org/x/net/context \
golang.org/x/net/context/ctxhttp \
golang.org/x/net/http2 \
diff --git a/cmd/coordinator/Makefile b/cmd/coordinator/Makefile
index 8a5126e..3030ce0 100644
--- a/cmd/coordinator/Makefile
+++ b/cmd/coordinator/Makefile
@@ -36,14 +36,9 @@
docker cp $(DOCKER_CTR_build0):/go/bin/$@ $@
docker rm $(DOCKER_CTR_build0)
-ca-certificates.crt:
- docker create --name $(DOCKER_CTR_build0) $(DOCKER_IMAGE_build0)
- docker cp $(DOCKER_CTR_build0):/etc/ssl/certs/$@ $@
- docker rm $(DOCKER_CTR_build0)
-
-docker-prod: Dockerfile coordinator ca-certificates.crt
+docker-prod: Dockerfile coordinator
docker build --force-rm --tag=gcr.io/symbolic-datum-552/coordinator:$(VERSION) .
-docker-dev: Dockerfile coordinator ca-certificates.crt
+docker-dev: Dockerfile coordinator
docker build --force-rm --tag=gcr.io/go-dashboard-dev/coordinator:latest .
push-prod: docker-prod
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
index f8ec446..eb4d6dd 100644
--- a/cmd/coordinator/coordinator.go
+++ b/cmd/coordinator/coordinator.go
@@ -321,6 +321,7 @@
}
go listenAndServeTLS()
+ go listenAndServeSSH() // ssh proxy to remote buildlets; remote.go
ticker := time.NewTicker(1 * time.Minute)
for {
diff --git a/cmd/coordinator/deployment-dev.yaml b/cmd/coordinator/deployment-dev.yaml
index d8c4070..85a2103 100644
--- a/cmd/coordinator/deployment-dev.yaml
+++ b/cmd/coordinator/deployment-dev.yaml
@@ -19,6 +19,7 @@
ports:
- containerPort: 80
- containerPort: 443
+ - containerPort: 2222 # ssh proxy port
resources:
requests:
cpu: "1"
diff --git a/cmd/coordinator/deployment-prod.yaml b/cmd/coordinator/deployment-prod.yaml
index 00e39e8..906c3c7 100644
--- a/cmd/coordinator/deployment-prod.yaml
+++ b/cmd/coordinator/deployment-prod.yaml
@@ -19,6 +19,7 @@
ports:
- containerPort: 80
- containerPort: 443
+ - containerPort: 2222 # ssh proxy port
resources:
requests:
cpu: "1"
diff --git a/cmd/coordinator/remote.go b/cmd/coordinator/remote.go
index fbda607..dbefffc 100644
--- a/cmd/coordinator/remote.go
+++ b/cmd/coordinator/remote.go
@@ -7,21 +7,39 @@
package main // import "golang.org/x/build/cmd/coordinator"
import (
+ "bufio"
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"html"
+ "io"
+ "io/ioutil"
"log"
+ "net"
"net/http"
"net/http/httputil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
"sort"
+ "strconv"
"strings"
"sync"
+ "syscall"
"time"
+ "unsafe"
+ "cloud.google.com/go/compute/metadata"
+
+ "github.com/gliderlabs/ssh"
+ "github.com/kr/pty"
"golang.org/x/build/buildlet"
"golang.org/x/build/dashboard"
+ "golang.org/x/build/internal/gophers"
+ gossh "golang.org/x/crypto/ssh"
)
var (
@@ -53,6 +71,22 @@
buildlet *buildlet.Client
}
+// renew renews rb's idle timeout if ctx hasn't expired.
+// renew should run in its own goroutine.
+func (rb *remoteBuildlet) renew(ctx context.Context) {
+ remoteBuildlets.Lock()
+ defer remoteBuildlets.Unlock()
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+ if got := remoteBuildlets.m[rb.Name]; got == rb {
+ rb.Expires = time.Now().Add(remoteBuildletIdleTimeout)
+ time.AfterFunc(time.Minute, func() { rb.renew(ctx) })
+ }
+}
+
func addRemoteBuildlet(rb *remoteBuildlet) (name string) {
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
@@ -305,9 +339,354 @@
return
}
if !strings.HasPrefix(user, "user-") || builderKey(user) != pass {
- http.Error(w, "bad username or password", 401)
- return
+ if *mode == "dev" {
+ log.Printf("ignoring gomote authentication failure for %q in dev mode", user)
+ } else {
+ http.Error(w, "bad username or password", 401)
+ return
+ }
}
h.ServeHTTP(w, r)
})
}
+
+var sshPrivateKeyFile string
+
+func writeSSHPrivateKeyToTempFile(key []byte) (path string, err error) {
+ tf, err := ioutil.TempFile("", "ssh-priv-key")
+ if err != nil {
+ return "", err
+ }
+ if err := tf.Chmod(0600); err != nil {
+ return "", err
+ }
+ if _, err := tf.Write(key); err != nil {
+ return "", err
+ }
+ return tf.Name(), tf.Close()
+}
+
+func listenAndServeSSH() {
+ const listenAddr = ":2222" // TODO: flag if ever necessary?
+ var hostKey []byte
+ var err error
+ if *mode == "dev" {
+ sshPrivateKeyFile = filepath.Join(os.Getenv("HOME"), "keys", "id_gomotessh_rsa")
+ hostKey, err = ioutil.ReadFile(sshPrivateKeyFile)
+ if os.IsNotExist(err) {
+ log.Printf("SSH host key file %s doesn't exist; not running SSH server.", sshPrivateKeyFile)
+ return
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+ } else {
+ if storageClient == nil {
+ log.Printf("GCS storage client not available; not running SSH server.")
+ return
+ }
+ r, err := storageClient.Bucket(buildEnv.BuildletBucket).Object("coordinator-gomote-ssh.key").NewReader(context.Background())
+ if err != nil {
+ log.Printf("Failed to read ssh host key: %v; not running SSH server.", err)
+ return
+ }
+ hostKey, err = ioutil.ReadAll(r)
+ if err != nil {
+ log.Printf("Failed to read ssh host key: %v; not running SSH server.", err)
+ return
+ }
+ sshPrivateKeyFile, err = writeSSHPrivateKeyToTempFile(hostKey)
+ log.Printf("ssh: writeSSHPrivateKeyToTempFile = %v, %v", sshPrivateKeyFile, err)
+ if err != nil {
+ log.Printf("error writing ssh private key to temp file: %v; not running SSH server", err)
+ return
+ }
+ }
+ signer, err := gossh.ParsePrivateKey(hostKey)
+ if err != nil {
+ log.Printf("failed to parse SSH host key: %v; running running SSH server", err)
+ return
+ }
+
+ s := &ssh.Server{
+ Addr: listenAddr,
+ Handler: handleIncomingSSHPostAuth,
+ PublicKeyHandler: handleSSHPublicKeyAuth,
+ }
+ s.AddHostKey(signer)
+
+ log.Printf("running SSH server on %s", listenAddr)
+ err = s.ListenAndServe()
+ log.Printf("SSH server ended with error: %v", err)
+ // TODO: make ListenAndServe errors Fatal, once it has a proven track record. starting paranoid.
+}
+
+func handleSSHPublicKeyAuth(ctx ssh.Context, key ssh.PublicKey) bool {
+ inst := ctx.User() // expected to be of form "user-USER-goos-goarch-etc"
+ user := userFromGomoteInstanceName(inst)
+ if user == "" {
+ return false
+ }
+ // Map the gomote username to the github username, and use the
+ // github user's public ssh keys for authentication. This is
+ // mostly of laziness and pragmatism, not wanting to invent or
+ // maintain a new auth mechanism or password/key registry.
+ githubUser := gophers.GithubOfGomoteUser(user)
+ keys := githubPublicKeys(githubUser)
+ for _, authKey := range keys {
+ if ssh.KeysEqual(key, authKey.PublicKey) {
+ log.Printf("for instance %q, github user %q key matched: %s", inst, githubUser, authKey.AuthorizedLine)
+ return true
+ }
+ }
+ return false
+}
+
+func handleIncomingSSHPostAuth(s ssh.Session) {
+ inst := s.User()
+ user := userFromGomoteInstanceName(inst)
+
+ requestedMutable := strings.HasPrefix(inst, "mutable-")
+ if requestedMutable {
+ inst = strings.TrimPrefix(inst, "mutable-")
+ }
+
+ ptyReq, winCh, isPty := s.Pty()
+ if !isPty {
+ fmt.Fprintf(s, "scp etc not yet supported; https://golang.org/issue/21140\n")
+ return
+ }
+
+ pubKey, err := metadata.ProjectAttributeValue("gomote-ssh-public-key")
+ if err != nil || pubKey == "" {
+ if err == nil {
+ err = errors.New("not found")
+ }
+ fmt.Fprintf(s, "failed to get GCE gomote-ssh-public-key: %v\n", err)
+ return
+ }
+
+ remoteBuildlets.Lock()
+ rb, ok := remoteBuildlets.m[inst]
+ remoteBuildlets.Unlock()
+ if !ok {
+ fmt.Fprintf(s, "unknown instance %q", inst)
+ return
+ }
+
+ hostType := rb.HostType
+ hostConf, ok := dashboard.Hosts[hostType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown host type %q\n", inst, hostType)
+ return
+ }
+
+ bconf, ok := dashboard.Builders[rb.BuilderType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown builder type %q\n", inst, rb.BuilderType)
+ return
+ }
+
+ ctx, cancel := context.WithCancel(s.Context())
+ defer cancel()
+ go rb.renew(ctx)
+
+ sshUser := hostConf.SSHUsername
+ needsSSHProxyPport := bconf.GOOS() != "plan9" && bconf.GOOS() != "windows"
+ if sshUser == "" && needsSSHProxyPport {
+ fmt.Fprintf(s, "instance %q host type %q does not have SSH configured\n", inst, hostType)
+ return
+ }
+ if !hostConf.IsHermetic() && !requestedMutable {
+ fmt.Fprintf(s, "WARNING: instance %q host type %q is not currently\n", inst, hostType)
+ fmt.Fprintf(s, "configured to have a hermetic filesystem per boot.\n")
+ fmt.Fprintf(s, "You must be careful not to modify machine state\n")
+ fmt.Fprintf(s, "that will affect future builds. Do you agree? If so,\n")
+ fmt.Fprintf(s, "run gomote ssh --i-will-not-break-the-host <INST>\n")
+ return
+ }
+
+ log.Printf("connecting to ssh to instance %q ...", inst)
+
+ fmt.Fprintf(s, "# Welcome to the gomote ssh proxy, %s.\n", user)
+ fmt.Fprintf(s, "# Connecting to/starting remote ssh...\n")
+ fmt.Fprintf(s, "#\n")
+
+ var localProxyPort int
+ if needsSSHProxyPport {
+ sshConn, err := rb.buildlet.ConnectSSH(sshUser, pubKey)
+ log.Printf("buildlet(%q).ConnectSSH = %T, %v", inst, sshConn, err)
+ if err != nil {
+ fmt.Fprintf(s, "failed to connect to ssh on %s: %v\n", inst, err)
+ return
+ }
+ defer sshConn.Close()
+
+ // Now listen on some localhost port that we'll proxy to sshConn.
+ // The openssh ssh command line tool will connect to this IP.
+ ln, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ fmt.Fprintf(s, "local listen error: %v\n", err)
+ return
+ }
+ localProxyPort = ln.Addr().(*net.TCPAddr).Port
+ log.Printf("ssh local proxy port for %s: %v", inst, localProxyPort)
+ var lnCloseOnce sync.Once
+ lnClose := func() { lnCloseOnce.Do(func() { ln.Close() }) }
+ defer lnClose()
+
+ // Accept at most one connection from localProxyPort and proxy
+ // it to sshConn.
+ go func() {
+ c, err := ln.Accept()
+ lnClose()
+ if err != nil {
+ return
+ }
+ defer c.Close()
+ errc := make(chan error, 1)
+ go func() {
+ _, err := io.Copy(c, sshConn)
+ errc <- err
+ }()
+ go func() {
+ _, err := io.Copy(sshConn, c)
+ errc <- err
+ }()
+ err = <-errc
+ }()
+ }
+ workDir, err := rb.buildlet.WorkDir()
+ if err != nil {
+ fmt.Fprintf(s, "Error getting WorkDir: %v\n", err)
+ return
+ }
+ ip, _, ipErr := net.SplitHostPort(rb.buildlet.IPPort())
+
+ fmt.Fprintf(s, "# `gomote push` and the builders use:\n")
+ fmt.Fprintf(s, "# - workdir: %s\n", workDir)
+ fmt.Fprintf(s, "# - GOROOT: %s/go\n", workDir)
+ fmt.Fprintf(s, "# - GOPATH: %s/gopath\n", workDir)
+ fmt.Fprintf(s, "# - env: %s\n", strings.Join(bconf.Env(), " ")) // TODO: shell quote?
+ fmt.Fprintf(s, "# Happy debugging.\n")
+
+ log.Printf("ssh to %s: starting ssh -p %d for %s@localhost", inst, localProxyPort, sshUser)
+ var cmd *exec.Cmd
+ switch runtime.GOOS {
+ default:
+ cmd = exec.Command("ssh",
+ "-p", strconv.Itoa(localProxyPort),
+ "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "StrictHostKeyChecking=no",
+ "-i", sshPrivateKeyFile,
+ sshUser+"@localhost")
+ case "windows":
+ fmt.Fprintf(s, "# Windows user/pass: gopher/gopher\n")
+ if ipErr != nil {
+ fmt.Fprintf(s, "# Failed to get IP out of %q: %v\n", rb.buildlet.IPPort(), err)
+ return
+ }
+ cmd = exec.Command("telnet", ip)
+ case "plan9":
+ fmt.Fprintf(s, "# Plan9 user/pass: glenda/glenda123\n")
+ if ipErr != nil {
+ fmt.Fprintf(s, "# Failed to get IP out of %q: %v\n", rb.buildlet.IPPort(), err)
+ return
+ }
+ cmd = exec.Command("/usr/local/bin/drawterm",
+ "-a", ip, "-c", ip, "-u", "glenda", "-k", "user=glenda")
+ }
+ cmd.Env = append(cmd.Env, fmt.Sprintf("TERM=%s", ptyReq.Term))
+ f, err := pty.Start(cmd)
+ if err != nil {
+ log.Printf("running ssh client to %s: %v", inst, err)
+ return
+ }
+ defer f.Close()
+ go func() {
+ for win := range winCh {
+ setWinsize(f, win.Width, win.Height)
+ }
+ }()
+ go func() {
+ io.Copy(f, s) // stdin
+ }()
+ io.Copy(s, f) // stdout
+ cmd.Process.Kill()
+ cmd.Wait()
+}
+
+func setWinsize(f *os.File, w, h int) {
+ syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
+ uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
+}
+
+// userFromGomoteInstanceName returns the username part of a gomote
+// remote instance name.
+//
+// The instance name is of two forms. The normal form is:
+//
+// user-bradfitz-linux-amd64-0
+//
+// The overloaded form to convey that the user accepts responsibility
+// for changes to the underlying host is to prefix the same instance
+// name with the string "mutable-", such as:
+//
+// mutable-user-bradfitz-darwin-amd64-10_8-0
+//
+// The mutable part is ignored by this function.
+func userFromGomoteInstanceName(name string) string {
+ name = strings.TrimPrefix(name, "mutable-")
+ if !strings.HasPrefix(name, "user-") {
+ return ""
+ }
+ user := name[len("user-"):]
+ hyphen := strings.IndexByte(user, '-')
+ if hyphen == -1 {
+ return ""
+ }
+ return user[:hyphen]
+}
+
+// authorizedKey is a Github user's SSH authorized key, in both string and parsed format.
+type authorizedKey struct {
+ AuthorizedLine string // e.g. "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILj8HGIG9NsT34PHxO8IBq0riSBv7snp30JM8AanBGoV"
+ PublicKey ssh.PublicKey
+}
+
+func githubPublicKeys(user string) []authorizedKey {
+ // TODO: caching, rate limiting.
+ req, err := http.NewRequest("GET", "https://github.com/"+user+".keys", nil)
+ if err != nil {
+ return nil
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ req = req.WithContext(ctx)
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ log.Printf("getting %s github keys: %v", user, err)
+ return nil
+ }
+ defer res.Body.Close()
+ if res.StatusCode != 200 {
+ return nil
+ }
+ var keys []authorizedKey
+ bs := bufio.NewScanner(res.Body)
+ for bs.Scan() {
+ key, _, _, _, err := ssh.ParseAuthorizedKey(bs.Bytes())
+ if err != nil {
+ log.Printf("parsing github user %q key %q: %v", user, bs.Text(), err)
+ continue
+ }
+ keys = append(keys, authorizedKey{
+ PublicKey: key,
+ AuthorizedLine: strings.TrimSpace(bs.Text()),
+ })
+ }
+ if err := bs.Err(); err != nil {
+ return nil
+ }
+ return keys
+}
diff --git a/cmd/coordinator/reverse.go b/cmd/coordinator/reverse.go
index a111e02..fc41364 100644
--- a/cmd/coordinator/reverse.go
+++ b/cmd/coordinator/reverse.go
@@ -518,6 +518,7 @@
},
},
})
+ client.SetDialer(revDialer.Dial)
client.SetDescription(fmt.Sprintf("reverse peer %s/%s for host type %v", hostname, r.RemoteAddr, hostType))
var isDead struct {
diff --git a/cmd/coordinator/service-dev.yaml b/cmd/coordinator/service-dev.yaml
index 8c50d58..e1f5e2e 100644
--- a/cmd/coordinator/service-dev.yaml
+++ b/cmd/coordinator/service-dev.yaml
@@ -10,6 +10,9 @@
- port: 443
targetPort: 443
name: https
+ - port: 2222
+ targetPort: 2222
+ name: ssh
selector:
app: coordinator
type: LoadBalancer
diff --git a/cmd/coordinator/service-prod.yaml b/cmd/coordinator/service-prod.yaml
index 24b99a2..d2a773e 100644
--- a/cmd/coordinator/service-prod.yaml
+++ b/cmd/coordinator/service-prod.yaml
@@ -10,6 +10,9 @@
- port: 443
targetPort: 443
name: https
+ - port: 2222
+ targetPort: 2222
+ name: ssh
selector:
app: coordinator
type: LoadBalancer
diff --git a/cmd/gomote/gomote.go b/cmd/gomote/gomote.go
index 61a3f2f..a04e1cc 100644
--- a/cmd/gomote/gomote.go
+++ b/cmd/gomote/gomote.go
@@ -35,6 +35,7 @@
puttar extract a tar.gz to a buildlet
rm delete files or directories
run run a command on a buildlet
+ ssh ssh to a buildlet
To list all the builder types available, run "create" with no arguments:
@@ -150,6 +151,7 @@
registerCommand("puttar", "extract a tar.gz to a buildlet", putTar)
registerCommand("rm", "delete files or directories", rm)
registerCommand("run", "run a command on a buildlet", run)
+ registerCommand("ssh", "ssh to a buildlet", ssh)
}
func main() {
diff --git a/cmd/gomote/ssh.go b/cmd/gomote/ssh.go
new file mode 100644
index 0000000..9a918fc
--- /dev/null
+++ b/cmd/gomote/ssh.go
@@ -0,0 +1,55 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+
+ "golang.org/x/build/internal/gophers"
+)
+
+func ssh(args []string) error {
+ fs := flag.NewFlagSet("ssh", flag.ContinueOnError)
+ fs.Usage = func() {
+ fmt.Fprintln(os.Stderr, "ssh usage: gomote ssh <instance>")
+ fs.PrintDefaults()
+ os.Exit(1)
+ }
+ var mutable bool
+ fs.BoolVar(&mutable, "i-will-not-break-the-host", false, "required for older host configs with reused filesystems; using this says that you are aware that your changes to the machine's root filesystem affect future builds. This is a no-op for the newer, safe host configs.")
+ fs.Parse(args)
+ if fs.NArg() != 1 {
+ fs.Usage()
+ }
+ name := fs.Arg(0)
+ _, _, err := clientAndConf(name)
+ if err != nil {
+ return err
+ }
+ // gomoteUser extracts "gopher" from "user-gopher-linux-amd64-0".
+ gomoteUser := strings.Split(name, "-")[1]
+ githubUser := gophers.GithubOfGomoteUser(gomoteUser)
+
+ sshUser := name
+ if mutable {
+ sshUser = "mutable-" + sshUser
+ }
+
+ ssh, err := exec.LookPath("ssh")
+ if err != nil {
+ log.Printf("No 'ssh' binary found in path so can't run:")
+ }
+ fmt.Printf("$ ssh -p 2222 %s@farmer.golang.org # auth using https://github.com/%s.keys\n", sshUser, githubUser)
+
+ // Best effort, where supported:
+ syscall.Exec(ssh, []string{"ssh", "-p", "2222", sshUser + "@farmer.golang.org"}, os.Environ())
+ return nil
+}
diff --git a/dashboard/builders.go b/dashboard/builders.go
index d1b7576..1dcd981 100644
--- a/dashboard/builders.go
+++ b/dashboard/builders.go
@@ -8,6 +8,7 @@
import (
"fmt"
+ "os"
"sort"
"strconv"
"strings"
@@ -29,6 +30,7 @@
KubeImage: "linux-x86-std-kube:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
+ SSHUsername: "root",
},
"host-linux-armhf-cross": &HostConfig{
Notes: "Kubernetes container on GKE built from env/crosscompile/linux-armhf-jessie",
@@ -42,7 +44,12 @@
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
-
+ "host-linux-amd64-localdev": &HostConfig{
+ IsReverse: true,
+ ExpectNum: 0,
+ Notes: "for localhost development of buildlets/gomote/coordinator only",
+ SSHUsername: os.Getenv("USER"),
+ },
"host-nacl-kube": &HostConfig{
Notes: "Kubernetes container on GKE.",
KubeImage: "linux-x86-nacl:latest",
@@ -74,10 +81,12 @@
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-arm-scaleway": &HostConfig{
- IsReverse: true,
- ExpectNum: 50,
- env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
- ReverseAliases: []string{"linux-arm", "linux-arm-arm5"},
+ IsReverse: true,
+ HermeticReverse: true,
+ ExpectNum: 50,
+ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
+ ReverseAliases: []string{"linux-arm", "linux-arm-arm5"},
+ SSHUsername: "root",
},
"host-linux-arm5spacemonkey": &HostConfig{
IsReverse: true,
@@ -92,6 +101,7 @@
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-60.tar.gz",
Notes: "OpenBSD 6.0; GCE VM is built from script in build/env/openbsd-amd64",
+ SSHUsername: "gopher",
},
"host-openbsd-386-60": &HostConfig{
VMImage: "openbsd-386-60",
@@ -99,12 +109,14 @@
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-60.tar.gz",
Notes: "OpenBSD 6.0; GCE VM is built from script in build/env/openbsd-386",
+ SSHUsername: "gopher",
},
"host-freebsd-93-gce": &HostConfig{
VMImage: "freebsd-amd64-gce93",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
+ SSHUsername: "gopher",
},
"host-freebsd-101-gce": &HostConfig{
VMImage: "freebsd-amd64-gce101",
@@ -113,6 +125,7 @@
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64", // TODO(bradfitz): why was this http instead of https?
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
env: []string{"CC=clang"},
+ SSHUsername: "gopher",
},
"host-freebsd-110": &HostConfig{
VMImage: "freebsd-amd64-110",
@@ -121,6 +134,7 @@
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64", // TODO(bradfitz): why was this http instead of https?
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
env: []string{"CC=clang"},
+ SSHUsername: "gopher",
},
"host-netbsd-8branch": &HostConfig{
VMImage: "netbsd-amd64-8branch",
@@ -128,9 +142,10 @@
machineType: "n1-highcpu-2",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.netbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-netbsd-amd64.tar.gz",
+ SSHUsername: "gopher",
},
"host-plan9-386-gce": &HostConfig{
- VMImage: "plan9-386-v4",
+ VMImage: "plan9-386-v5",
Notes: "Plan 9 from 0intro; GCE VM is built from script in build/env/plan9-386",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.plan9-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-plan9-386.tar.gz",
@@ -185,7 +200,9 @@
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
- ReverseAliases: []string{"darwin-amd64-10_8"},
+ ReverseAliases: []string{"darwin-amd64-10_8"},
+ SSHUsername: "gopher",
+ HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_10": &HostConfig{
IsReverse: true,
@@ -194,7 +211,9 @@
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
- ReverseAliases: []string{"darwin-amd64-10_10"},
+ ReverseAliases: []string{"darwin-amd64-10_10"},
+ SSHUsername: "gopher",
+ HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_11": &HostConfig{
IsReverse: true,
@@ -203,7 +222,9 @@
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
- ReverseAliases: []string{"darwin-amd64-10_11"},
+ ReverseAliases: []string{"darwin-amd64-10_11"},
+ SSHUsername: "gopher",
+ HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_12": &HostConfig{
IsReverse: true,
@@ -212,7 +233,9 @@
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
- ReverseAliases: []string{"darwin-amd64-10_12"},
+ ReverseAliases: []string{"darwin-amd64-10_12"},
+ SSHUsername: "gopher",
+ HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-linux-s390x": &HostConfig{
Notes: "run by IBM",
@@ -222,31 +245,39 @@
ReverseAliases: []string{"linux-s390x-ibm"},
},
"host-linux-ppc64-osu": &HostConfig{
- Notes: "Debian jessie; run by Go team on osuosl.org",
- IsReverse: true,
- ExpectNum: 5,
- env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
- ReverseAliases: []string{"linux-ppc64-buildlet"},
+ Notes: "Debian jessie; run by Go team on osuosl.org",
+ IsReverse: true,
+ ExpectNum: 5,
+ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
+ ReverseAliases: []string{"linux-ppc64-buildlet"},
+ SSHUsername: "debian",
+ HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
},
"host-linux-ppc64le-osu": &HostConfig{
- Notes: "Debian jessie; run by Go team on osuosl.org",
- IsReverse: true,
- ExpectNum: 5,
- env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
- ReverseAliases: []string{"linux-ppc64le-buildlet"},
+ Notes: "Debian jessie; run by Go team on osuosl.org",
+ IsReverse: true,
+ ExpectNum: 5,
+ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
+ ReverseAliases: []string{"linux-ppc64le-buildlet"},
+ SSHUsername: "debian",
+ HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
},
"host-linux-arm64-linaro": &HostConfig{
- Notes: "Ubuntu xenial; run by Go team, from linaro",
- IsReverse: true,
- ExpectNum: 5,
- env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
- ReverseAliases: []string{"linux-arm64-buildlet"},
+ Notes: "Ubuntu xenial; run by Go team, from linaro",
+ IsReverse: true,
+ HermeticReverse: true,
+ ExpectNum: 5,
+ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
+ ReverseAliases: []string{"linux-arm64-buildlet"},
+ SSHUsername: "root",
},
"host-linux-arm64-packet": &HostConfig{
- Notes: "On 96 core packet.net host (Xenial) in Docker containers (Jessie); run by Go team. See x/build/env/linux-arm64/packet",
- IsReverse: true,
- ExpectNum: 20,
- env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
+ Notes: "On 96 core packet.net host (Xenial) in Docker containers (Jessie); run by Go team. See x/build/env/linux-arm64/packet",
+ IsReverse: true,
+ HermeticReverse: true,
+ ExpectNum: 20,
+ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
+ SSHUsername: "root",
},
"host-solaris-amd64": &HostConfig{
Notes: "run by Go team on Joyent, on a SmartOS 'infrastructure container'",
@@ -401,7 +432,8 @@
RegularDisk bool // if true, use spinning disk instead of SSD
// ReverseOptions:
- ExpectNum int // expected number of reverse buildlets of this type
+ ExpectNum int // expected number of reverse buildlets of this type
+ HermeticReverse bool // whether reverse buildlet has fresh env per conn
// Optional base env. GOROOT_BOOTSTRAP should go here if the buildlet
// has Go 1.4+ baked in somewhere.
@@ -415,6 +447,8 @@
OwnerGithub string // optional GitHub username of owner
Notes string // notes for humans
+ SSHUsername string // username to ssh as, empty means not supported
+
// ReverseAliases lists alternate names for this buildlet
// config, for older clients doing a reverse dial into the
// coordinator from outside. This prevents us from updating
@@ -736,7 +770,23 @@
case c.IsKube():
return "Kubernetes container"
}
- return "??"
+ panic("unknown builder type")
+}
+
+// IsHermetic reports whether this host config gets a fresh
+// environment (including /usr, /var, etc) for each execution. This is
+// true for VMs, GKE, and reverse buildlets running their containers
+// running in Docker, but false on some reverse buildlets.
+func (c *HostConfig) IsHermetic() bool {
+ switch {
+ case c.IsReverse:
+ return c.HermeticReverse
+ case c.IsGCE():
+ return true
+ case c.IsKube():
+ return true
+ }
+ panic("unknown builder type")
}
// GCENumCPU reports the number of GCE CPUs this buildlet requires.
@@ -1173,6 +1223,12 @@
"CC_FOR_TARGET=s390x-linux-gnu-gcc",
},
})
+ addBuilder(BuildConfig{
+ Name: "linux-amd64-localdev",
+ HostType: "host-linux-amd64-localdev",
+ Notes: "for localhost development only",
+ TryOnly: true,
+ })
}
func (c BuildConfig) isMobile() bool {
diff --git a/env/linux-arm64/Dockerfile b/env/linux-arm64/Dockerfile
index fe6e18a..a486773 100644
--- a/env/linux-arm64/Dockerfile
+++ b/env/linux-arm64/Dockerfile
@@ -10,6 +10,7 @@
apt-get install --yes \
curl gcc strace ca-certificates \
procps lsof psmisc
+RUN apt-get install --yes --no-install-recommends openssh-server
RUN mkdir /usr/local/go-bootstrap && \
curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-arm64.tar.gz | \
diff --git a/env/linux-x86-std-kube/Dockerfile b/env/linux-x86-std-kube/Dockerfile
index 042e30f..2f91a72 100644
--- a/env/linux-x86-std-kube/Dockerfile
+++ b/env/linux-x86-std-kube/Dockerfile
@@ -32,6 +32,7 @@
libgles2-mesa-dev \
libopenal-dev \
fonts-droid \
+ openssh-server \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir -p /go1.4-amd64 \
diff --git a/env/plan9-386/README b/env/plan9-386/README
index f1e99d6..416281f 100644
--- a/env/plan9-386/README
+++ b/env/plan9-386/README
@@ -19,3 +19,15 @@
Also, due to an ATA bug affecting QEMU 1.6 and 1.7, the
Plan 9 CD can't be booted with these versions.
+
+To create the image:
+
+Then:
+ $ gsutil cp -a public-read plan9-386-gce.tar.gz gs://go-builder-data/plan9-386-gce.tar.gz
+
+Then:
+ $ gcloud compute --project symbolic-datum-552 images create plan9-386-v5 --source-uri gs://go-builder-data/plan9-386-gce.tar.gz
+
+And optional optimization for faster boots:
+ $ go install golang.org/x/build/cmd/coordinator/buildongce
+ $ buildongce -make-basepin
diff --git a/internal/gophers/gophers.go b/internal/gophers/gophers.go
index 59b86fa..36ca5f7 100644
--- a/internal/gophers/gophers.go
+++ b/internal/gophers/gophers.go
@@ -1544,3 +1544,14 @@
addPerson("Фахриддин Балтаев", "faxriddinjon@gmail.com", "@faxriddin")
addPerson("张嵩", "zs349596@gmail.com", "@zs1379")
}
+
+// GithubOfGomoteUser returns the GitHub username for the provided gomote user.
+func GithubOfGomoteUser(gomoteUser string) (githubUser string) {
+ switch gomoteUser {
+ case "austin":
+ return "aclements"
+ case "herbie":
+ return "cybrcodr"
+ }
+ return gomoteUser
+}
diff --git a/revdial/revdial.go b/revdial/revdial.go
index 23d1858..ea56799 100644
--- a/revdial/revdial.go
+++ b/revdial/revdial.go
@@ -234,29 +234,87 @@
w *bufio.Writer
unregConn func(id uint32) // called with wmu held
- mu sync.Mutex
- cond *sync.Cond
- buf []byte // unread data
- eof bool // remote side closed
- closed bool // our side closed (with Close)
+ mu sync.Mutex
+ cond *sync.Cond
+ buf []byte // unread data
+ eof bool // remote side closed
+ closed bool // our side closed (with Close)
+ rdeadline time.Time
+ wdeadline time.Time
+ rtimer *time.Timer
+ wtimer *time.Timer
}
var errUnsupported = errors.New("revdial: unsupported Conn operation")
-func (c *conn) SetDeadline(t time.Time) error { return errUnsupported }
-func (c *conn) SetReadDeadline(t time.Time) error { return errUnsupported }
-func (c *conn) SetWriteDeadline(t time.Time) error { return errUnsupported }
-func (c *conn) LocalAddr() net.Addr { return fakeAddr{} }
-func (c *conn) RemoteAddr() net.Addr { return fakeAddr{} }
+func (c *conn) LocalAddr() net.Addr { return fakeAddr{} }
+func (c *conn) RemoteAddr() net.Addr { return fakeAddr{} }
+
+func (c *conn) SetDeadline(t time.Time) error {
+ rerr := c.SetReadDeadline(t)
+ werr := c.SetWriteDeadline(t)
+ if rerr != nil {
+ return rerr
+ }
+ return werr
+}
+
+func (c *conn) SetWriteDeadline(t time.Time) error {
+ defer c.cond.Signal()
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.closed {
+ return errors.New("SetWriteDeadline called on closed connection")
+ }
+ c.stopWriteTimerLocked()
+ c.wdeadline = t
+ now := time.Now()
+ if t.After(now) {
+ c.wtimer = time.AfterFunc(t.Sub(now), c.cond.Broadcast)
+ }
+ return nil
+}
+
+func (c *conn) SetReadDeadline(t time.Time) error {
+ defer c.cond.Signal()
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.closed {
+ return errors.New("SetReadDeadline called on closed connection")
+ }
+ c.stopReadTimerLocked()
+ c.rdeadline = t
+ now := time.Now()
+ if t.After(now) {
+ c.rtimer = time.AfterFunc(t.Sub(now), c.cond.Broadcast)
+ }
+ return nil
+}
+
+func (c *conn) stopReadTimerLocked() {
+ if c.rtimer != nil {
+ c.rtimer.Stop()
+ c.rtimer = nil
+ }
+}
+
+func (c *conn) stopWriteTimerLocked() {
+ if c.wtimer != nil {
+ c.wtimer.Stop()
+ c.wtimer = nil
+ }
+}
func (c *conn) Close() error {
+ defer c.cond.Broadcast()
c.mu.Lock()
if c.closed {
c.mu.Unlock()
return nil
}
+ c.stopReadTimerLocked()
+ c.stopWriteTimerLocked()
c.closed = true
- c.cond.Signal()
c.mu.Unlock()
c.wmu.Lock()
@@ -273,37 +331,52 @@
}
func (c *conn) peerWrite(p []byte) (n int, err error) {
+ defer c.cond.Signal()
c.mu.Lock()
defer c.mu.Unlock()
- defer c.cond.Signal()
// TODO(bradfitz): bound this, like http2's buffer/pipe code
c.buf = append(c.buf, p...)
return len(p), nil
}
func (c *conn) peerClose() {
+ defer c.cond.Broadcast()
c.mu.Lock()
defer c.mu.Unlock()
- defer c.cond.Broadcast()
c.eof = true
}
+var errDeadline net.Error = deadlineError{}
+
+type deadlineError struct{}
+
+func (deadlineError) Error() string { return "revdial: Read/Write deadline expired" }
+func (deadlineError) Temporary() bool { return false }
+func (deadlineError) Timeout() bool { return true }
+
func (c *conn) Read(p []byte) (n int, err error) {
+ defer c.cond.Signal() // for when writers block
c.mu.Lock()
defer c.mu.Unlock()
- defer c.cond.Signal() // for when writers block
- for len(c.buf) == 0 && !c.eof && !c.closed {
+ for {
+ n = copy(p, c.buf)
+ c.buf = c.buf[:copy(c.buf, c.buf[n:])] // slide down
+ if dl := c.rdeadline; !dl.IsZero() {
+ if time.Now().After(dl) {
+ return n, errDeadline
+ }
+ }
+ if c.closed {
+ return n, errors.New("revdial: Read on closed connection")
+ }
+ if len(c.buf) == 0 && c.eof {
+ return n, io.EOF
+ }
+ if n > 0 || len(p) == 0 {
+ return n, nil
+ }
c.cond.Wait()
}
- if c.closed {
- return 0, errors.New("revdial: Read on closed connection")
- }
- if len(c.buf) == 0 && c.eof {
- return 0, io.EOF
- }
- n = copy(p, c.buf)
- c.buf = c.buf[:copy(c.buf, c.buf[n:])] // slide down
- return n, nil
}
func (c *conn) Write(p []byte) (n int, err error) {
@@ -312,28 +385,58 @@
c.mu.Unlock()
return 0, errors.New("revdial: Write on Closed conn")
}
+ dl := c.wdeadline
+ if !dl.IsZero() && time.Now().After(dl) {
+ c.mu.Unlock()
+ // TODO: better write deadline support. do it per chunk, push it down
+ // to underlying net.Conn (which involves changing API to let caller
+ // supply a net.Conn)
+ return 0, errDeadline
+ }
c.mu.Unlock()
- const max = 0xffff // max chunk size
- for len(p) > 0 {
- chunk := p
- if len(chunk) > max {
- chunk = chunk[:max]
- }
- c.wmu.Lock()
- err = writeFrame(c, frame{
- command: frameWrite,
- connID: c.id,
- payload: chunk,
- })
- c.wmu.Unlock()
- if err != nil {
- return n, err
- }
- n += len(chunk)
- p = p[len(chunk):]
+ var timeout <-chan time.Time
+ if !dl.IsZero() {
+ timer := time.NewTimer(dl.Sub(time.Now()))
+ defer timer.Stop()
+ timeout = timer.C
}
- return n, nil
+ type result struct {
+ n int
+ err error
+ }
+ res := make(chan result, 1)
+ go func() {
+ const max = 0xffff // max chunk size
+ n := 0
+ for len(p) > 0 {
+ chunk := p
+ if len(chunk) > max {
+ chunk = chunk[:max]
+ }
+ c.wmu.Lock()
+ err = writeFrame(c, frame{
+ command: frameWrite,
+ connID: c.id,
+ payload: chunk,
+ })
+ c.wmu.Unlock()
+ if err != nil {
+ res <- result{n, err}
+ return
+ }
+ n += len(chunk)
+ p = p[len(chunk):]
+ }
+ res <- result{n: n}
+ }()
+ select {
+ case v := <-res:
+ return v.n, v.err
+ case <-timeout:
+ println("timeout for " + dl.String())
+ return 0, errDeadline
+ }
}
type frameType uint8
diff --git a/revdial/revdial_test.go b/revdial/revdial_test.go
index a32401b..840716e 100644
--- a/revdial/revdial_test.go
+++ b/revdial/revdial_test.go
@@ -13,6 +13,8 @@
"sync"
"testing"
"time"
+
+ "golang.org/x/net/nettest"
)
func TestDialer(t *testing.T) {
@@ -226,3 +228,75 @@
t.Error("timeout waiting for Done channel")
}
}
+
+func TestConnAgainstNetTest(t *testing.T) {
+ if testing.Short() {
+ t.Skipf("testing in short mode")
+ }
+ t.Logf("warning: the revdial's SetWriteDeadline support is not complete; some tests involving write deadlines known to be flaky")
+ nettest.TestConn(t, func() (c1, c2 net.Conn, stop func(), err error) {
+ tln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ return
+ }
+ cc, err := net.Dial("tcp", tln.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ return
+ }
+ sc, err := tln.Accept()
+ if err != nil {
+ t.Fatal(err)
+ return
+ }
+
+ rd := NewDialer(bufio.NewReadWriter(
+ bufio.NewReader(sc),
+ bufio.NewWriter(sc),
+ ), ioutil.NopCloser(nil))
+
+ rl := NewListener(bufio.NewReadWriter(
+ bufio.NewReader(cc),
+ bufio.NewWriter(cc),
+ ))
+
+ c1c := make(chan interface{}, 1)
+ c2c := make(chan interface{}, 1)
+ go func() {
+ c, err := rd.Dial()
+ if err != nil {
+ c1c <- err
+ return
+ }
+ c1c <- c
+ }()
+ go func() {
+ c, err := rl.Accept()
+ if err != nil {
+ c2c <- err
+ return
+ }
+ c2c <- c
+ }()
+ switch v := (<-c1c).(type) {
+ case net.Conn:
+ c1 = v
+ case error:
+ t.Fatalf("revdial.Dial: %v", v)
+ }
+ switch v := (<-c2c).(type) {
+ case net.Conn:
+ c2 = v
+ case error:
+ t.Fatalf("revdial.Accept: %v", v)
+ }
+
+ stop = func() {
+ tln.Close()
+ cc.Close()
+ sc.Close()
+ }
+ return
+ })
+}
