buildlet: don't leak file descriptors to buildlets
Also, add count of fds and goroutines to the coordinator's status
page.
Change-Id: I857e609623cfa280716d5d079180d0e4021d0bac
Reviewed-on: https://go-review.googlesource.com/27550
Reviewed-by: Quentin Smith <quentin@golang.org>
diff --git a/buildlet/buildletclient.go b/buildlet/buildletclient.go
index 83fefe1..ef38a61 100644
--- a/buildlet/buildletclient.go
+++ b/buildlet/buildletclient.go
@@ -31,16 +31,17 @@
//
// This constructor returns immediately without testing the host or auth.
func NewClient(ipPort string, kp KeyPair) *Client {
+ tr := &http.Transport{
+ Dial: defaultDialer(),
+ DialTLS: kp.tlsDialer(),
+ IdleConnTimeout: time.Minute,
+ }
c := &Client{
- ipPort: ipPort,
- tls: kp,
- password: kp.Password(),
- httpClient: &http.Client{
- Transport: &http.Transport{
- Dial: defaultDialer(),
- DialTLS: kp.tlsDialer(),
- },
- },
+ ipPort: ipPort,
+ tls: kp,
+ password: kp.Password(),
+ httpClient: &http.Client{Transport: tr},
+ closeFuncs: []func(){tr.CloseIdleConnections},
}
c.setCommon()
return c
@@ -83,6 +84,9 @@
if err == nil {
err = ErrClosed
}
+ for _, fn := range c.closeFuncs {
+ fn()
+ }
c.setPeerDead(err) // which will also cause c.heartbeatFailure to run
})
return nil
@@ -133,6 +137,8 @@
password string // basic auth password or empty for none
remoteBuildlet string // non-empty if for remote buildlets
+ closeFuncs []func() // optional extra code to run on close
+
ctx context.Context
ctxCancel context.CancelFunc
heartbeatFailure func() // optional
diff --git a/cmd/coordinator/status.go b/cmd/coordinator/status.go
index 78cf610..f2a09a7 100644
--- a/cmd/coordinator/status.go
+++ b/cmd/coordinator/status.go
@@ -8,8 +8,11 @@
"bytes"
"fmt"
"html/template"
+ "io"
"net/http"
+ "os"
"os/exec"
+ "runtime"
"sort"
"strings"
"time"
@@ -27,11 +30,13 @@
statusMu.Lock()
data := statusData{
- Total: len(status),
- Uptime: round(time.Now().Sub(processStartTime)),
- Recent: append([]*buildStatus{}, statusDone...),
- DiskFree: df,
- Version: Version,
+ Total: len(status),
+ Uptime: round(time.Now().Sub(processStartTime)),
+ Recent: append([]*buildStatus{}, statusDone...),
+ DiskFree: df,
+ Version: Version,
+ NumFD: fdCount(),
+ NumGoroutine: runtime.NumGoroutine(),
}
for _, st := range status {
data.Active = append(data.Active, st)
@@ -85,6 +90,25 @@
buf.WriteTo(w)
}
+func fdCount() int {
+ f, err := os.Open("/proc/self/fd")
+ if err != nil {
+ return -1
+ }
+ defer f.Close()
+ n := 0
+ for {
+ names, err := f.Readdirnames(1000)
+ n += len(names)
+ if err == io.EOF {
+ return n
+ }
+ if err != nil {
+ return -1
+ }
+ }
+}
+
func diskFree() string {
out, _ := exec.Command("df", "-h").Output()
return string(out)
@@ -92,7 +116,9 @@
// statusData is the data that fills out statusTmpl.
type statusData struct {
- Total int
+ Total int // number of total builds active
+ NumFD int
+ NumGoroutine int
Uptime time.Duration
Active []*buildStatus
Recent []*buildStatus
@@ -157,6 +183,12 @@
<h2 id=disk><a href='#disk'>🔗</a> Disk Space</h2>
<pre>{{.DiskFree}}</pre>
+<h2 id=disk><a href='#fd'>🔗</a> File Descriptors</h2>
+<p>{{.NumFD}}</p>
+
+<h2 id=disk><a href='#goroutines'>🔗</a> Goroutines</h2>
+<p>{{.NumGoroutine}} <a href='/debug/goroutines'>goroutines</a></p>
+
</body>
</html>
`))
