dashboard/coordinator: run background goroutine cleaning old docker containers
I'm tired of figuring out what isn't cleaning up after itself, so keep
a background goroutine that looks at old containers and deletes them
as a backup measure. Verified it works by creating some dummy containers on
the machine.
Also adds df output to the HTML status page.
Change-Id: I23adc22872def882b3b9b3a4ec730017899bb966
Reviewed-on: https://go-review.googlesource.com/1537
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/coordinator/main.go b/coordinator/main.go
index f8db7d1..3820668 100644
--- a/coordinator/main.go
+++ b/coordinator/main.go
@@ -12,6 +12,7 @@
"encoding/json"
"flag"
"fmt"
+ "html"
"io"
"io/ioutil"
"log"
@@ -129,6 +130,8 @@
http.HandleFunc("/logs", handleLogs)
go http.ListenAndServe(":80", nil)
+ go cleanUpOldContainers()
+
for _, watcher := range watchers {
if err := startWatching(watchers[watcher.repo]); err != nil {
log.Printf("Error starting watcher for %s: %v", watcher.repo, err)
@@ -218,7 +221,12 @@
fmt.Fprintf(w, "%-22s hg %s in container <a href='/logs?name=%s&rev=%s'>%s</a>, %v ago\n", st.name, st.rev, st.name, st.rev,
st.container, time.Now().Sub(st.start))
}
- fmt.Fprintf(w, "</pre></body></html>")
+ fmt.Fprintf(w, "</pre><h2>disk space</h2><pre>%s</pre></body></html>", html.EscapeString(diskFree()))
+}
+
+func diskFree() string {
+ out, _ := exec.Command("df", "-h").Output()
+ return string(out)
}
func handleLogs(w http.ResponseWriter, r *http.Request) {
@@ -558,3 +566,18 @@
}
masterKeyCache = bytes.TrimSpace(slurp)
}
+
+func cleanUpOldContainers() {
+ for {
+ for _, cid := range oldContainers() {
+ log.Printf("Cleaning old container %v", cid)
+ exec.Command("docker", "rm", "-v", cid).Run()
+ }
+ time.Sleep(30 * time.Second)
+ }
+}
+
+func oldContainers() []string {
+ out, _ := exec.Command("docker", "ps", "-a", "--filter=status=exited", "--no-trunc", "-q").Output()
+ return strings.Fields(string(out))
+}
