cmd/coordinator, internal/coordinator/remote: move post handlers
This change moves the SSH handlers into the internal packages. It also
adds the handler which will use the session pool instead of the remote
buildlets.
Updates golang/go#52594
For golang/go#47521
Change-Id: I7e99fdbb16e0f80a871696cec79a9b638354e662
Reviewed-on: https://go-review.googlesource.com/c/build/+/405257
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Carlos Amedee <carlos@golang.org>
Reviewed-by: Carlos Amedee <carlos@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Run-TryBot: Carlos Amedee <carlos@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
index c2aaff3..9354c67 100644
--- a/cmd/coordinator/coordinator.go
+++ b/cmd/coordinator/coordinator.go
@@ -406,16 +406,7 @@
if err != nil {
return nil, fmt.Errorf("unable to retrieve keys for SSH Server: %v", err)
}
- privateHostKeyFile, err := remote.WriteSSHPrivateKeyToTempFile(privateKey)
- log.Printf("unable to write private host key file: %s", err)
- if err != nil {
- return nil, fmt.Errorf("error writing ssh private key to temp file: %v; not running SSH server", err)
- }
- sshHandlers := &sshHandlers{
- gomotePublicKey: string(publicKey),
- sshPrivateKeyFile: privateHostKeyFile,
- }
- return remote.NewSSHServer(*sshAddr, privateKey, sshCA, sshHandlers.handleIncomingSSHPostAuth, sp)
+ return remote.NewSSHServer(*sshAddr, privateKey, publicKey, sshCA, sp, remoteBuildlets)
}
sshServ, err := configureSSHServer()
if err != nil {
diff --git a/cmd/coordinator/remote.go b/cmd/coordinator/remote.go
index 48003f5..9e0fad7 100644
--- a/cmd/coordinator/remote.go
+++ b/cmd/coordinator/remote.go
@@ -12,7 +12,6 @@
import (
"bytes"
- "context"
"encoding/json"
"fmt"
"html"
@@ -21,32 +20,23 @@
"net"
"net/http"
"net/http/httputil"
- "os"
- "os/exec"
"sort"
"strconv"
"strings"
- "sync"
- "syscall"
"time"
- "unsafe"
- "github.com/creack/pty"
- "github.com/gliderlabs/ssh"
"golang.org/x/build/buildlet"
"golang.org/x/build/dashboard"
"golang.org/x/build/internal/coordinator/pool"
"golang.org/x/build/internal/coordinator/remote"
"golang.org/x/build/internal/coordinator/schedule"
- "golang.org/x/build/internal/envutil"
"golang.org/x/build/types"
)
var (
- remoteBuildlets = struct {
- sync.Mutex
- m map[string]*remoteBuildlet // keyed by buildletName
- }{m: map[string]*remoteBuildlet{}}
+ remoteBuildlets = &remote.Buildlets{
+ M: map[string]*remote.Buildlet{},
+ }
cleanTimer *time.Timer
)
@@ -60,43 +50,16 @@
cleanTimer = time.AfterFunc(remoteBuildletCleanInterval, expireBuildlets)
}
-type remoteBuildlet struct {
- User string // "user-foo" build key
- Name string // dup of key
- HostType string
- BuilderType string // default builder config to use if not overwritten
- Created time.Time
- Expires time.Time
-
- buildlet buildlet.Client
-}
-
-// renew renews rb's idle timeout if ctx hasn't expired.
-// renew should run in its own goroutine.
-func (rb *remoteBuildlet) renew(ctx context.Context) {
- remoteBuildlets.Lock()
- defer remoteBuildlets.Unlock()
- select {
- case <-ctx.Done():
- return
- default:
- }
- if got := remoteBuildlets.m[rb.Name]; got == rb {
- rb.Expires = time.Now().Add(remoteBuildletIdleTimeout)
- time.AfterFunc(time.Minute, func() { rb.renew(ctx) })
- }
-}
-
-func addRemoteBuildlet(rb *remoteBuildlet) (name string) {
+func addRemoteBuildlet(rb *remote.Buildlet) (name string) {
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
n := 0
for {
name = fmt.Sprintf("%s-%s-%d", rb.User, rb.BuilderType, n)
- if _, ok := remoteBuildlets.m[name]; ok {
+ if _, ok := remoteBuildlets.M[name]; ok {
n++
} else {
- remoteBuildlets.m[name] = rb
+ remoteBuildlets.M[name] = rb
return name
}
}
@@ -105,8 +68,8 @@
func isGCERemoteBuildlet(instName string) bool {
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
- for _, rb := range remoteBuildlets.m {
- if rb.buildlet.GCEInstanceName() == instName {
+ for _, rb := range remoteBuildlets.M {
+ if rb.Buildlet().GCEInstanceName() == instName {
return true
}
}
@@ -118,10 +81,10 @@
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
now := time.Now()
- for name, rb := range remoteBuildlets.m {
+ for name, rb := range remoteBuildlets.M {
if !rb.Expires.IsZero() && rb.Expires.Before(now) {
- go rb.buildlet.Close()
- delete(remoteBuildlets.m, name)
+ go rb.Buildlet().Close()
+ delete(remoteBuildlets.M, name)
}
}
}
@@ -194,7 +157,7 @@
// One of these fields is set:
type msg struct {
Error string `json:"error,omitempty"`
- Buildlet *remoteBuildlet `json:"buildlet,omitempty"`
+ Buildlet *remote.Buildlet `json:"buildlet,omitempty"`
Status *types.BuildletWaitStatus `json:"status,omitempty"`
}
sendJSONLine := func(v interface{}) {
@@ -237,14 +200,14 @@
sendJSONLine(msg{Status: &st})
case bc := <-resc:
now := timeNow()
- rb := &remoteBuildlet{
+ rb := &remote.Buildlet{
User: user,
BuilderType: builderType,
HostType: bconf.HostType,
- buildlet: bc,
Created: now,
Expires: now.Add(remoteBuildletIdleTimeout),
}
+ rb.SetBuildlet(bc)
rb.Name = addRemoteBuildlet(rb)
bc.SetName(rb.Name)
log.Printf("created buildlet %v for %v (%s)", rb.Name, rb.User, bc.String())
@@ -279,11 +242,11 @@
http.Error(w, "GET required", 400)
return
}
- res := make([]*remoteBuildlet, 0) // so it's never JSON "null"
+ res := make([]*remote.Buildlet, 0) // so it's never JSON "null"
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
user, _, _ := r.BasicAuth()
- for _, rb := range remoteBuildlets.m {
+ for _, rb := range remoteBuildlets.M {
if rb.User == user {
res = append(res, rb)
}
@@ -299,7 +262,7 @@
w.Write(jenc)
}
-type byBuildletName []*remoteBuildlet
+type byBuildletName []*remote.Buildlet
func (s byBuildletName) Len() int { return len(s) }
func (s byBuildletName) Less(i, j int) bool { return s[i].Name < s[j].Name }
@@ -309,13 +272,13 @@
remoteBuildlets.Lock()
defer remoteBuildlets.Unlock()
- if len(remoteBuildlets.m) == 0 {
+ if len(remoteBuildlets.M) == 0 {
return "<i>(none)</i>"
}
var buf bytes.Buffer
- var all []*remoteBuildlet
- for _, rb := range remoteBuildlets.m {
+ var all []*remote.Buildlet
+ for _, rb := range remoteBuildlets.M {
all = append(all, rb)
}
sort.Sort(byBuildletName(all))
@@ -342,7 +305,7 @@
return
}
remoteBuildlets.Lock()
- rb, ok := remoteBuildlets.m[buildletName]
+ rb, ok := remoteBuildlets.M[buildletName]
if ok {
rb.Expires = time.Now().Add(remoteBuildletIdleTimeout)
}
@@ -358,13 +321,13 @@
}
if r.Method == "POST" && r.URL.Path == "/halt" {
- err := rb.buildlet.Close()
+ err := rb.Buildlet().Close()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
- rb.buildlet.Close()
+ rb.Buildlet().Close()
remoteBuildlets.Lock()
- delete(remoteBuildlets.m, buildletName)
+ delete(remoteBuildlets.M, buildletName)
remoteBuildlets.Unlock()
return
}
@@ -375,7 +338,7 @@
return
}
- outReq, err := http.NewRequest(r.Method, rb.buildlet.URL()+r.URL.Path+"?"+r.URL.RawQuery, r.Body)
+ outReq, err := http.NewRequest(r.Method, rb.Buildlet().URL()+r.URL.Path+"?"+r.URL.RawQuery, r.Body)
if err != nil {
log.Printf("bad proxy request: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -385,7 +348,7 @@
outReq.ContentLength = r.ContentLength
proxy := &httputil.ReverseProxy{
Director: func(*http.Request) {}, // nothing
- Transport: rb.buildlet.ProxyRoundTripper(),
+ Transport: rb.Buildlet().ProxyRoundTripper(),
FlushInterval: 500 * time.Millisecond,
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
log.Printf("gomote proxy error for %s: %v", buildletName, err)
@@ -399,7 +362,7 @@
// proxyBuildletTCP handles connecting to and proxying between a
// backend buildlet VM's TCP port and the client. This is called once
// it's already authenticated by proxyBuildletHTTP.
-func proxyBuildletTCP(w http.ResponseWriter, r *http.Request, rb *remoteBuildlet) {
+func proxyBuildletTCP(w http.ResponseWriter, r *http.Request, rb *remote.Buildlet) {
if r.ProtoMajor > 1 {
// TODO: deal with HTTP/2 requests if https://farmer.golang.org enables it later.
// Currently it does not, as other handlers Hijack too. We'd need to teach clients
@@ -426,9 +389,9 @@
http.Error(w, fmt.Sprintf("unsupported non-VM host type %q", rb.HostType), http.StatusBadRequest)
return
}
- ip, _, err := net.SplitHostPort(rb.buildlet.IPPort())
+ ip, _, err := net.SplitHostPort(rb.Buildlet().IPPort())
if err != nil {
- http.Error(w, fmt.Sprintf("unexpected backend ip:port %q", rb.buildlet.IPPort()), http.StatusInternalServerError)
+ http.Error(w, fmt.Sprintf("unexpected backend ip:port %q", rb.Buildlet().IPPort()), http.StatusInternalServerError)
return
}
@@ -495,176 +458,3 @@
h.ServeHTTP(w, r)
})
}
-
-type sshHandlers struct {
- gomotePublicKey string
- sshPrivateKeyFile string
-}
-
-func (ah *sshHandlers) handleIncomingSSHPostAuth(s ssh.Session) {
- inst := s.User()
- user := remote.UserFromGomoteInstanceName(inst)
-
- requestedMutable := strings.HasPrefix(inst, "mutable-")
- if requestedMutable {
- inst = strings.TrimPrefix(inst, "mutable-")
- }
-
- ptyReq, winCh, isPty := s.Pty()
- if !isPty {
- fmt.Fprintf(s, "scp etc not yet supported; https://golang.org/issue/21140\n")
- return
- }
-
- if ah.gomotePublicKey == "" {
- fmt.Fprint(s, "invalid gomote-ssh-public-key")
- return
- }
-
- remoteBuildlets.Lock()
- rb, ok := remoteBuildlets.m[inst]
- remoteBuildlets.Unlock()
- if !ok {
- fmt.Fprintf(s, "unknown instance %q", inst)
- return
- }
-
- hostType := rb.HostType
- hostConf, ok := dashboard.Hosts[hostType]
- if !ok {
- fmt.Fprintf(s, "instance %q has unknown host type %q\n", inst, hostType)
- return
- }
-
- bconf, ok := dashboard.Builders[rb.BuilderType]
- if !ok {
- fmt.Fprintf(s, "instance %q has unknown builder type %q\n", inst, rb.BuilderType)
- return
- }
-
- ctx, cancel := context.WithCancel(s.Context())
- defer cancel()
- go rb.renew(ctx)
-
- sshUser := hostConf.SSHUsername
- useLocalSSHProxy := bconf.GOOS() != "plan9"
- if sshUser == "" && useLocalSSHProxy {
- fmt.Fprintf(s, "instance %q host type %q does not have SSH configured\n", inst, hostType)
- return
- }
- if !hostConf.IsHermetic() && !requestedMutable {
- fmt.Fprintf(s, "WARNING: instance %q host type %q is not currently\n", inst, hostType)
- fmt.Fprintf(s, "configured to have a hermetic filesystem per boot.\n")
- fmt.Fprintf(s, "You must be careful not to modify machine state\n")
- fmt.Fprintf(s, "that will affect future builds. Do you agree? If so,\n")
- fmt.Fprintf(s, "run gomote ssh --i-will-not-break-the-host <INST>\n")
- return
- }
-
- log.Printf("connecting to ssh to instance %q ...", inst)
-
- fmt.Fprintf(s, "# Welcome to the gomote ssh proxy, %s.\n", user)
- fmt.Fprintf(s, "# Connecting to/starting remote ssh...\n")
- fmt.Fprintf(s, "#\n")
-
- var localProxyPort int
- if useLocalSSHProxy {
- sshConn, err := rb.buildlet.ConnectSSH(sshUser, ah.gomotePublicKey)
- log.Printf("buildlet(%q).ConnectSSH = %T, %v", inst, sshConn, err)
- if err != nil {
- fmt.Fprintf(s, "failed to connect to ssh on %s: %v\n", inst, err)
- return
- }
- defer sshConn.Close()
-
- // Now listen on some localhost port that we'll proxy to sshConn.
- // The openssh ssh command line tool will connect to this IP.
- ln, err := net.Listen("tcp", "localhost:0")
- if err != nil {
- fmt.Fprintf(s, "local listen error: %v\n", err)
- return
- }
- localProxyPort = ln.Addr().(*net.TCPAddr).Port
- log.Printf("ssh local proxy port for %s: %v", inst, localProxyPort)
- var lnCloseOnce sync.Once
- lnClose := func() { lnCloseOnce.Do(func() { ln.Close() }) }
- defer lnClose()
-
- // Accept at most one connection from localProxyPort and proxy
- // it to sshConn.
- go func() {
- c, err := ln.Accept()
- lnClose()
- if err != nil {
- return
- }
- defer c.Close()
- errc := make(chan error, 1)
- go func() {
- _, err := io.Copy(c, sshConn)
- errc <- err
- }()
- go func() {
- _, err := io.Copy(sshConn, c)
- errc <- err
- }()
- err = <-errc
- }()
- }
- workDir, err := rb.buildlet.WorkDir(ctx)
- if err != nil {
- fmt.Fprintf(s, "Error getting WorkDir: %v\n", err)
- return
- }
- ip, _, ipErr := net.SplitHostPort(rb.buildlet.IPPort())
-
- fmt.Fprintf(s, "# `gomote push` and the builders use:\n")
- fmt.Fprintf(s, "# - workdir: %s\n", workDir)
- fmt.Fprintf(s, "# - GOROOT: %s/go\n", workDir)
- fmt.Fprintf(s, "# - GOPATH: %s/gopath\n", workDir)
- fmt.Fprintf(s, "# - env: %s\n", strings.Join(bconf.Env(), " ")) // TODO: shell quote?
- fmt.Fprintf(s, "# Happy debugging.\n")
-
- log.Printf("ssh to %s: starting ssh -p %d for %s@localhost", inst, localProxyPort, sshUser)
- var cmd *exec.Cmd
- switch bconf.GOOS() {
- default:
- cmd = exec.Command("ssh",
- "-p", strconv.Itoa(localProxyPort),
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "StrictHostKeyChecking=no",
- "-i", ah.sshPrivateKeyFile,
- sshUser+"@localhost")
- case "plan9":
- fmt.Fprintf(s, "# Plan9 user/pass: glenda/glenda123\n")
- if ipErr != nil {
- fmt.Fprintf(s, "# Failed to get IP out of %q: %v\n", rb.buildlet.IPPort(), err)
- return
- }
- cmd = exec.Command("/usr/local/bin/drawterm",
- "-a", ip, "-c", ip, "-u", "glenda", "-k", "user=glenda")
- }
- envutil.SetEnv(cmd, "TERM="+ptyReq.Term)
- f, err := pty.Start(cmd)
- if err != nil {
- log.Printf("running ssh client to %s: %v", inst, err)
- return
- }
- defer f.Close()
- go func() {
- for win := range winCh {
- setWinsize(f, win.Width, win.Height)
- }
- }()
- go func() {
- io.Copy(f, s) // stdin
- }()
- io.Copy(s, f) // stdout
- cmd.Process.Kill()
- cmd.Wait()
-}
-
-func setWinsize(f *os.File, w, h int) {
- syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
- uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
-}
diff --git a/cmd/coordinator/remote_test.go b/cmd/coordinator/remote_test.go
index 9ee0207..a6893a3 100644
--- a/cmd/coordinator/remote_test.go
+++ b/cmd/coordinator/remote_test.go
@@ -23,6 +23,7 @@
"golang.org/x/build/buildlet"
"golang.org/x/build/dashboard"
"golang.org/x/build/internal/coordinator/pool"
+ "golang.org/x/build/internal/coordinator/remote"
"golang.org/x/build/internal/gophers"
)
@@ -125,7 +126,7 @@
log.SetOutput(tlogger{t})
defer log.SetOutput(os.Stderr)
addBuilder(buildName)
- remoteBuildlets.m = map[string]*remoteBuildlet{}
+ remoteBuildlets.M = map[string]*remote.Buildlet{}
pool.TestPoolHook = func(_ *dashboard.HostConfig) pool.Buildlet { return testPool }
defer func() {
timeNow = time.Now
@@ -154,7 +155,7 @@
log.SetOutput(tlogger{t})
defer log.SetOutput(os.Stderr)
addBuilder(buildName)
- remoteBuildlets.m = map[string]*remoteBuildlet{}
+ remoteBuildlets.M = map[string]*remote.Buildlet{}
pool.TestPoolHook = func(_ *dashboard.HostConfig) pool.Buildlet { return testPool }
defer func() {
timeNow = time.Now
diff --git a/internal/coordinator/remote/legacy.go b/internal/coordinator/remote/legacy.go
new file mode 100644
index 0000000..8db61ee
--- /dev/null
+++ b/internal/coordinator/remote/legacy.go
@@ -0,0 +1,57 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package remote
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "golang.org/x/build/buildlet"
+)
+
+// Buildlets is a store for the legacy remote buildlets.
+type Buildlets struct {
+ sync.Mutex
+ M map[string]*Buildlet // keyed by buildletName
+}
+
+// Buildlet is the representation of the legacy remote buildlet.
+type Buildlet struct {
+ User string // "user-foo" build key
+ Name string // dup of key
+ HostType string
+ BuilderType string // default builder config to use if not overwritten
+ Created time.Time
+ Expires time.Time
+
+ buildlet buildlet.Client
+}
+
+// Renew renews rb's idle timeout if ctx hasn't expired.
+// Renew should run in its own goroutine.
+func (rb *Buildlet) Renew(ctx context.Context, rbs *Buildlets) {
+ rbs.Lock()
+ defer rbs.Unlock()
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+ if got := rbs.M[rb.Name]; got == rb {
+ rb.Expires = time.Now().Add(remoteBuildletIdleTimeout)
+ time.AfterFunc(time.Minute, func() { rb.Renew(ctx, rbs) })
+ }
+}
+
+// Buildlet returns the buildlet client for the associated legacy buildlet.
+func (rb *Buildlet) Buildlet() buildlet.Client {
+ return rb.buildlet
+}
+
+// SetBuildlet sets the buildlet client for a legacy buildlet.
+func (rb *Buildlet) SetBuildlet(b buildlet.Client) {
+ rb.buildlet = b
+}
diff --git a/internal/coordinator/remote/ssh.go b/internal/coordinator/remote/ssh.go
index aabca55..7064157 100644
--- a/internal/coordinator/remote/ssh.go
+++ b/internal/coordinator/remote/ssh.go
@@ -2,6 +2,10 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+//go:build go1.16 && (linux || darwin)
+// +build go1.16
+// +build linux darwin
+
package remote
import (
@@ -14,14 +18,24 @@
"crypto/x509"
"encoding/pem"
"fmt"
+ "io"
"io/ioutil"
"log"
"net"
"net/http"
+ "os"
+ "os/exec"
+ "strconv"
"strings"
+ "sync"
+ "syscall"
"time"
+ "unsafe"
+ "github.com/creack/pty"
gssh "github.com/gliderlabs/ssh"
+ "golang.org/x/build/dashboard"
+ "golang.org/x/build/internal/envutil"
"golang.org/x/build/internal/gophers"
"golang.org/x/crypto/ssh"
)
@@ -83,27 +97,40 @@
// SSHServer is the SSH server that the coordinator provides.
type SSHServer struct {
- server *gssh.Server
+ gomotePublicKey string
+ privateHostKeyFile string
+ remoteBuildlets *Buildlets
+ server *gssh.Server
+ sessionPool *SessionPool
}
// NewSSHServer creates an SSH server used to access remote buildlet sessions.
-func NewSSHServer(addr string, hostPrivateKey, caPrivateKey []byte, handler gssh.Handler, sp *SessionPool) (*SSHServer, error) {
+func NewSSHServer(addr string, hostPrivateKey, gomotePublicKey, caPrivateKey []byte, sp *SessionPool, rbs *Buildlets) (*SSHServer, error) {
hostSigner, err := ssh.ParsePrivateKey(hostPrivateKey)
if err != nil {
- return nil, fmt.Errorf("failed to parse SSH host key: %v; not running SSH server", err)
+ return nil, fmt.Errorf("failed to parse SSH host key: %v; not configuring SSH server", err)
}
CASigner, err := ssh.ParsePrivateKey(caPrivateKey)
if err != nil {
- return nil, fmt.Errorf("failed to parse SSH host key: %v; not running SSH server", err)
+ return nil, fmt.Errorf("failed to parse SSH host key: %v; not configuring SSH server", err)
}
- return &SSHServer{
+ privateHostKeyFile, err := WriteSSHPrivateKeyToTempFile(hostPrivateKey)
+ if err != nil {
+ return nil, fmt.Errorf("error writing ssh private key to temp file: %v; not configuring SSH server", err)
+ }
+ s := &SSHServer{
+ gomotePublicKey: string(gomotePublicKey),
+ privateHostKeyFile: privateHostKeyFile,
+ remoteBuildlets: rbs,
+ sessionPool: sp,
server: &gssh.Server{
Addr: addr,
- Handler: handler,
PublicKeyHandler: chainSSHPublicKeyHandlers(handleSSHPublicKeyAuth, handleCertificateAuthFunc(sp, CASigner)),
HostSigners: []gssh.Signer{hostSigner},
},
- }, nil
+ }
+ s.server.Handler = s.HandleIncomingSSHPostAuth
+ return s, nil
}
// ListenAndServe attempts to start the SSH server. This blocks until the server stops.
@@ -122,6 +149,340 @@
return ss.server.Serve(l)
}
+// HandleIncomingSSHPostAuth handles SSH requests after the session has been authenticated. It forwards
+// the request to either the legacy handler or the new handler.
+// TODO(go.dev/issue/47521) The legacy path can be removed once the migration to the new security model
+// is complete.
+func (ss *SSHServer) HandleIncomingSSHPostAuth(s gssh.Session) {
+ inst := s.User()
+ requestedMutable := strings.HasPrefix(inst, "mutable-")
+ if requestedMutable {
+ inst = strings.TrimPrefix(inst, "mutable-")
+ }
+ _, _, isPty := s.Pty()
+ if !isPty {
+ fmt.Fprintf(s, "scp etc not yet supported; https://golang.org/issue/21140\n")
+ return
+ }
+ if ss.gomotePublicKey == "" {
+ fmt.Fprint(s, "invalid gomote-ssh-public-key")
+ return
+ }
+ ss.remoteBuildlets.Lock()
+ rb, ok := ss.remoteBuildlets.M[inst]
+ ss.remoteBuildlets.Unlock()
+ if ok {
+ ss.legacyIncomingSSHPostAuth(s, rb)
+ return
+ }
+ rs, err := ss.sessionPool.Session(inst)
+ if err != nil {
+ fmt.Fprintf(s, "unknown instance %q", inst)
+ return
+ }
+ ss.IncomingSSHPostAuth(s, rs)
+}
+
+// legacyIncomingSSHPostAuth is the handler for legacy remote buildlet SSH sessions.
+func (ss *SSHServer) legacyIncomingSSHPostAuth(s gssh.Session, rb *Buildlet) {
+ inst := s.User()
+ user := UserFromGomoteInstanceName(inst)
+
+ requestedMutable := strings.HasPrefix(inst, "mutable-")
+ if requestedMutable {
+ inst = strings.TrimPrefix(inst, "mutable-")
+ }
+
+ ptyReq, winCh, isPty := s.Pty()
+ if !isPty {
+ fmt.Fprintf(s, "scp etc not yet supported; https://golang.org/issue/21140\n")
+ return
+ }
+
+ if ss.gomotePublicKey == "" {
+ fmt.Fprint(s, "invalid gomote-ssh-public-key")
+ return
+ }
+
+ hostType := rb.HostType
+ hostConf, ok := dashboard.Hosts[hostType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown host type %q\n", inst, hostType)
+ return
+ }
+
+ bconf, ok := dashboard.Builders[rb.BuilderType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown builder type %q\n", inst, rb.BuilderType)
+ return
+ }
+
+ ctx, cancel := context.WithCancel(s.Context())
+ defer cancel()
+ go rb.Renew(ctx, ss.remoteBuildlets)
+
+ sshUser := hostConf.SSHUsername
+ useLocalSSHProxy := bconf.GOOS() != "plan9"
+ if sshUser == "" && useLocalSSHProxy {
+ fmt.Fprintf(s, "instance %q host type %q does not have SSH configured\n", inst, hostType)
+ return
+ }
+ if !hostConf.IsHermetic() && !requestedMutable {
+ fmt.Fprintf(s, "WARNING: instance %q host type %q is not currently\n", inst, hostType)
+ fmt.Fprintf(s, "configured to have a hermetic filesystem per boot.\n")
+ fmt.Fprintf(s, "You must be careful not to modify machine state\n")
+ fmt.Fprintf(s, "that will affect future builds. Do you agree? If so,\n")
+ fmt.Fprintf(s, "run gomote ssh --i-will-not-break-the-host <INST>\n")
+ return
+ }
+
+ log.Printf("connecting to ssh to instance %q ...", inst)
+
+ fmt.Fprintf(s, "# Welcome to the gomote ssh proxy, %s.\n", user)
+ fmt.Fprintf(s, "# Connecting to/starting remote ssh...\n")
+ fmt.Fprintf(s, "#\n")
+
+ var localProxyPort int
+ if useLocalSSHProxy {
+ sshConn, err := rb.Buildlet().ConnectSSH(sshUser, ss.gomotePublicKey)
+ log.Printf("buildlet(%q).ConnectSSH = %T, %v", inst, sshConn, err)
+ if err != nil {
+ fmt.Fprintf(s, "failed to connect to ssh on %s: %v\n", inst, err)
+ return
+ }
+ defer sshConn.Close()
+
+ // Now listen on some localhost port that we'll proxy to sshConn.
+ // The openssh ssh command line tool will connect to this IP.
+ ln, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ fmt.Fprintf(s, "local listen error: %v\n", err)
+ return
+ }
+ localProxyPort = ln.Addr().(*net.TCPAddr).Port
+ log.Printf("ssh local proxy port for %s: %v", inst, localProxyPort)
+ var lnCloseOnce sync.Once
+ lnClose := func() { lnCloseOnce.Do(func() { ln.Close() }) }
+ defer lnClose()
+
+ // Accept at most one connection from localProxyPort and proxy
+ // it to sshConn.
+ go func() {
+ c, err := ln.Accept()
+ lnClose()
+ if err != nil {
+ return
+ }
+ defer c.Close()
+ errc := make(chan error, 1)
+ go func() {
+ _, err := io.Copy(c, sshConn)
+ errc <- err
+ }()
+ go func() {
+ _, err := io.Copy(sshConn, c)
+ errc <- err
+ }()
+ err = <-errc
+ }()
+ }
+ workDir, err := rb.Buildlet().WorkDir(ctx)
+ if err != nil {
+ fmt.Fprintf(s, "Error getting WorkDir: %v\n", err)
+ return
+ }
+ ip, _, ipErr := net.SplitHostPort(rb.Buildlet().IPPort())
+
+ fmt.Fprintf(s, "# `gomote push` and the builders use:\n")
+ fmt.Fprintf(s, "# - workdir: %s\n", workDir)
+ fmt.Fprintf(s, "# - GOROOT: %s/go\n", workDir)
+ fmt.Fprintf(s, "# - GOPATH: %s/gopath\n", workDir)
+ fmt.Fprintf(s, "# - env: %s\n", strings.Join(bconf.Env(), " ")) // TODO: shell quote?
+ fmt.Fprintf(s, "# Happy debugging.\n")
+
+ log.Printf("ssh to %s: starting ssh -p %d for %s@localhost", inst, localProxyPort, sshUser)
+ var cmd *exec.Cmd
+ switch bconf.GOOS() {
+ default:
+ cmd = exec.Command("ssh",
+ "-p", strconv.Itoa(localProxyPort),
+ "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "StrictHostKeyChecking=no",
+ "-i", ss.privateHostKeyFile,
+ sshUser+"@localhost")
+ case "plan9":
+ fmt.Fprintf(s, "# Plan9 user/pass: glenda/glenda123\n")
+ if ipErr != nil {
+ fmt.Fprintf(s, "# Failed to get IP out of %q: %v\n", rb.Buildlet().IPPort(), err)
+ return
+ }
+ cmd = exec.Command("/usr/local/bin/drawterm",
+ "-a", ip, "-c", ip, "-u", "glenda", "-k", "user=glenda")
+ }
+ envutil.SetEnv(cmd, "TERM="+ptyReq.Term)
+ f, err := pty.Start(cmd)
+ if err != nil {
+ log.Printf("running ssh client to %s: %v", inst, err)
+ return
+ }
+ defer f.Close()
+ go func() {
+ for win := range winCh {
+ setWinsize(f, win.Width, win.Height)
+ }
+ }()
+ go func() {
+ io.Copy(f, s) // stdin
+ }()
+ io.Copy(s, f) // stdout
+ cmd.Process.Kill()
+ cmd.Wait()
+}
+
+// IncomingSSHPostAuth handles post-authentication requests for the SSH server. This handler uses
+// Sessions for session management.
+func (ss *SSHServer) IncomingSSHPostAuth(s gssh.Session, rs *Session) {
+ inst := s.User()
+ ptyReq, winCh, _ := s.Pty()
+ hostConf, ok := dashboard.Hosts[rs.HostType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown host type %q\n", inst, rs.HostType)
+ return
+ }
+ bconf, ok := dashboard.Builders[rs.BuilderType]
+ if !ok {
+ fmt.Fprintf(s, "instance %q has unknown builder type %q\n", inst, rs.BuilderType)
+ return
+ }
+
+ ctx, cancel := context.WithCancel(s.Context())
+ defer cancel()
+ err := ss.sessionPool.KeepAlive(ctx, inst)
+ if err != nil {
+ log.Printf("ssh: KeepAlive on session=%s failed: %s", inst, err)
+ }
+
+ sshUser := hostConf.SSHUsername
+ useLocalSSHProxy := bconf.GOOS() != "plan9"
+ if sshUser == "" && useLocalSSHProxy {
+ fmt.Fprintf(s, "instance %q host type %q does not have SSH configured\n", inst, rs.HostType)
+ return
+ }
+ if !hostConf.IsHermetic() {
+ fmt.Fprintf(s, "WARNING: instance %q host type %q is not currently\n", inst, rs.HostType)
+ fmt.Fprintf(s, "configured to have a hermetic filesystem per boot.\n")
+ fmt.Fprintf(s, "You must be careful not to modify machine state\n")
+ fmt.Fprintf(s, "that will affect future builds. Do you agree? If so,\n")
+ fmt.Fprintf(s, "run gomote ssh --i-will-not-break-the-host <INST>\n")
+ return
+ }
+ log.Printf("connecting to ssh to instance %q ...", inst)
+ fmt.Fprint(s, "# Welcome to the gomote ssh proxy.\n")
+ fmt.Fprint(s, "# Connecting to/starting remote ssh...\n")
+ fmt.Fprint(s, "#\n")
+
+ var localProxyPort int
+ bc, err := ss.sessionPool.BuildletClient(inst)
+ if err != nil {
+ fmt.Fprintf(s, "failed to connect to ssh on %s: %v\n", inst, err)
+ return
+ }
+ if useLocalSSHProxy {
+ sshConn, err := bc.ConnectSSH(sshUser, ss.gomotePublicKey)
+ log.Printf("buildlet(%q).ConnectSSH = %T, %v", inst, sshConn, err)
+ if err != nil {
+ fmt.Fprintf(s, "failed to connect to ssh on %s: %v\n", inst, err)
+ return
+ }
+ defer sshConn.Close()
+
+ // Now listen on some localhost port that we'll proxy to sshConn.
+ // The openssh ssh command line tool will connect to this IP.
+ ln, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ fmt.Fprintf(s, "local listen error: %v\n", err)
+ return
+ }
+ localProxyPort = ln.Addr().(*net.TCPAddr).Port
+ log.Printf("ssh local proxy port for %s: %v", inst, localProxyPort)
+ var lnCloseOnce sync.Once
+ lnClose := func() { lnCloseOnce.Do(func() { ln.Close() }) }
+ defer lnClose()
+
+ // Accept at most one connection from localProxyPort and proxy
+ // it to sshConn.
+ go func() {
+ c, err := ln.Accept()
+ lnClose()
+ if err != nil {
+ return
+ }
+ defer c.Close()
+ errc := make(chan error, 1)
+ go func() {
+ _, err := io.Copy(c, sshConn)
+ errc <- err
+ }()
+ go func() {
+ _, err := io.Copy(sshConn, c)
+ errc <- err
+ }()
+ err = <-errc
+ }()
+ }
+ workDir, err := bc.WorkDir(ctx)
+ if err != nil {
+ fmt.Fprintf(s, "Error getting WorkDir: %v\n", err)
+ return
+ }
+ ip, _, ipErr := net.SplitHostPort(bc.IPPort())
+
+ fmt.Fprint(s, "# `gomote push` and the builders use:\n")
+ fmt.Fprintf(s, "# - workdir: %s\n", workDir)
+ fmt.Fprintf(s, "# - GOROOT: %s/go\n", workDir)
+ fmt.Fprintf(s, "# - GOPATH: %s/gopath\n", workDir)
+ fmt.Fprintf(s, "# - env: %s\n", strings.Join(bconf.Env(), " ")) // TODO: shell quote?
+ fmt.Fprint(s, "# Happy debugging.\n")
+
+ log.Printf("ssh to %s: starting ssh -p %d for %s@localhost", inst, localProxyPort, sshUser)
+ var cmd *exec.Cmd
+ switch bconf.GOOS() {
+ default:
+ cmd = exec.Command("ssh",
+ "-p", strconv.Itoa(localProxyPort),
+ "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "StrictHostKeyChecking=no",
+ "-i", ss.privateHostKeyFile,
+ sshUser+"@localhost")
+ case "plan9":
+ fmt.Fprintf(s, "# Plan9 user/pass: glenda/glenda123\n")
+ if ipErr != nil {
+ fmt.Fprintf(s, "# Failed to get IP out of %q: %v\n", bc.IPPort(), err)
+ return
+ }
+ cmd = exec.Command("/usr/local/bin/drawterm",
+ "-a", ip, "-c", ip, "-u", "glenda", "-k", "user=glenda")
+ }
+ envutil.SetEnv(cmd, "TERM="+ptyReq.Term)
+ f, err := pty.Start(cmd)
+ if err != nil {
+ log.Printf("running ssh client to %s: %v", inst, err)
+ return
+ }
+ defer f.Close()
+ go func() {
+ for win := range winCh {
+ setWinsize(f, win.Width, win.Height)
+ }
+ }()
+ go func() {
+ io.Copy(f, s) // stdin
+ }()
+ io.Copy(s, f) // stdout
+ cmd.Process.Kill()
+ cmd.Wait()
+}
+
// WriteSSHPrivateKeyToTempFile writes a key to a temporary file on the local file system. It also
// sets the permissions on the file to what is expected by OpenSSH implementations of SSH.
func WriteSSHPrivateKeyToTempFile(key []byte) (path string, err error) {
@@ -276,3 +637,8 @@
}
return user[:hyphen]
}
+
+func setWinsize(f *os.File, w, h int) {
+ syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
+ uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
+}
diff --git a/internal/coordinator/remote/ssh_test.go b/internal/coordinator/remote/ssh_test.go
index cd75c21..45d89fb 100644
--- a/internal/coordinator/remote/ssh_test.go
+++ b/internal/coordinator/remote/ssh_test.go
@@ -2,6 +2,10 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+//go:build go1.16 && (linux || darwin)
+// +build go1.16
+// +build linux darwin
+
package remote
import (
@@ -10,7 +14,6 @@
"testing"
"time"
- gssh "github.com/gliderlabs/ssh"
"github.com/google/go-cmp/cmp"
"golang.org/x/build/buildlet"
"golang.org/x/crypto/ssh"
@@ -230,11 +233,10 @@
t.Fatalf("nettest.NewLocalListener(tcp) = _, %s; want no error", err)
}
addr = l.Addr().String()
- h := func(ses gssh.Session) {
- t.Logf("look at my session=%+v", ses)
- return
+ rbs := &Buildlets{
+ M: map[string]*Buildlet{},
}
- s, err = NewSSHServer(addr, []byte(devCertAlternateClientPrivate), []byte(devCertCAPrivate), h, sp)
+ s, err = NewSSHServer(addr, []byte(devCertAlternateClientPrivate), []byte(devCertCAPublic), []byte(devCertCAPrivate), sp, rbs)
if err != nil {
t.Fatalf("NewSSHServer(...) = %s; want no error", err)
}
