buildlet: don't leak file descriptors to buildlets

Also, add count of fds and goroutines to the coordinator's status
page.

Change-Id: I857e609623cfa280716d5d079180d0e4021d0bac
Reviewed-on: https://go-review.googlesource.com/27550
Reviewed-by: Quentin Smith <[email protected]>

Author: bradfitz

buildlet/buildletclient.go

@@ -31,16 +31,17 @@ import (
 //
 // This constructor returns immediately without testing the host or auth.
 func NewClient(ipPort string, kp KeyPair) *Client {
+	tr := &http.Transport{
+		Dial:            defaultDialer(),
+		DialTLS:         kp.tlsDialer(),
+		IdleConnTimeout: time.Minute,
+	}
 	c := &Client{
-		ipPort:   ipPort,
-		tls:      kp,
-		password: kp.Password(),
-		httpClient: &http.Client{
-			Transport: &http.Transport{
-				Dial:    defaultDialer(),
-				DialTLS: kp.tlsDialer(),
-			},
-		},
+		ipPort:     ipPort,
+		tls:        kp,
+		password:   kp.Password(),
+		httpClient: &http.Client{Transport: tr},
+		closeFuncs: []func(){tr.CloseIdleConnections},
 	}
 	c.setCommon()
 	return c
@@ -83,6 +84,9 @@ func (c *Client) Close() error {
 		if err == nil {
 			err = ErrClosed
 		}
+		for _, fn := range c.closeFuncs {
+			fn()
+		}
 		c.setPeerDead(err) // which will also cause c.heartbeatFailure to run
 	})
 	return nil
@@ -133,6 +137,8 @@ type Client struct {
 	password       string // basic auth password or empty for none
 	remoteBuildlet string // non-empty if for remote buildlets
 
+	closeFuncs []func() // optional extra code to run on close
+
 	ctx              context.Context
 	ctxCancel        context.CancelFunc
 	heartbeatFailure func() // optional

cmd/coordinator/status.go

@@ -8,8 +8,11 @@ import (
 	"bytes"
 	"fmt"
 	"html/template"
+	"io"
 	"net/http"
+	"os"
 	"os/exec"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -27,11 +30,13 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 
 	statusMu.Lock()
 	data := statusData{
-		Total:    len(status),
-		Uptime:   round(time.Now().Sub(processStartTime)),
-		Recent:   append([]*buildStatus{}, statusDone...),
-		DiskFree: df,
-		Version:  Version,
+		Total:        len(status),
+		Uptime:       round(time.Now().Sub(processStartTime)),
+		Recent:       append([]*buildStatus{}, statusDone...),
+		DiskFree:     df,
+		Version:      Version,
+		NumFD:        fdCount(),
+		NumGoroutine: runtime.NumGoroutine(),
 	}
 	for _, st := range status {
 		data.Active = append(data.Active, st)
@@ -85,14 +90,35 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 	buf.WriteTo(w)
 }
 
+func fdCount() int {
+	f, err := os.Open("/proc/self/fd")
+	if err != nil {
+		return -1
+	}
+	defer f.Close()
+	n := 0
+	for {
+		names, err := f.Readdirnames(1000)
+		n += len(names)
+		if err == io.EOF {
+			return n
+		}
+		if err != nil {
+			return -1
+		}
+	}
+}
+
 func diskFree() string {
 	out, _ := exec.Command("df", "-h").Output()
 	return string(out)
 }
 
 // statusData is the data that fills out statusTmpl.
 type statusData struct {
-	Total             int
+	Total             int // number of total builds active
+	NumFD             int
+	NumGoroutine      int
 	Uptime            time.Duration
 	Active            []*buildStatus
 	Recent            []*buildStatus
@@ -157,6 +183,12 @@ var statusTmpl = template.Must(template.New("status").Parse(`
 <h2 id=disk><a href='#disk'>🔗</a> Disk Space</h2>
 <pre>{{.DiskFree}}</pre>
 
+<h2 id=disk><a href='#fd'>🔗</a> File Descriptors</h2>
+<p>{{.NumFD}}</p>
+
+<h2 id=disk><a href='#goroutines'>🔗</a> Goroutines</h2>
+<p>{{.NumGoroutine}} <a href='/debug/goroutines'>goroutines</a></p>
+
 </body>
 </html>
 `))