devapp: add stats page plus some minor cleanup

+ Adds a stats page that shows open CLs over time
+ Adds a command-line flag to reload templates on each request
+ Breaks some repeated code into some shared logic
+ Some minor cleanup to adhere to best practices

Change-Id: I2d84a1e5c77c7e9131c758d69e4a9bf9a9d815f4
Reviewed-on: https://go-review.googlesource.com/c/build/+/199637
Run-TryBot: Andrew Bonventre <andybons@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alexander Rakoczy <alex@golang.org>
diff --git a/devapp/data.go b/devapp/data.go
new file mode 100644
index 0000000..28ccbba
--- /dev/null
+++ b/devapp/data.go
@@ -0,0 +1,50 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"golang.org/x/build/maintner"
+)
+
+var (
+	excludedProjects = map[string]bool{
+		"gocloud":              true,
+		"google-api-go-client": true,
+	}
+	deletedChanges = map[struct {
+		proj string
+		num  int32
+	}]bool{
+		{"crypto", 35958}:  true,
+		{"scratch", 71730}: true,
+		{"scratch", 71850}: true,
+		{"scratch", 72090}: true,
+		{"scratch", 72091}: true,
+		{"scratch", 72110}: true,
+		{"scratch", 72131}: true,
+		{"tools", 93515}:   true,
+	}
+)
+
+func filterProjects(fn func(*maintner.GerritProject) error) func(*maintner.GerritProject) error {
+	return func(p *maintner.GerritProject) error {
+		if excludedProjects[p.Project()] {
+			return nil
+		}
+		return fn(p)
+	}
+}
+
+func withoutDeletedCLs(p *maintner.GerritProject, fn func(*maintner.GerritCL) error) func(*maintner.GerritCL) error {
+	return func(cl *maintner.GerritCL) error {
+		if deletedChanges[struct {
+			proj string
+			num  int32
+		}{p.Project(), cl.Number}] {
+			return nil
+		}
+		return fn(cl)
+	}
+}
diff --git a/devapp/devapp.go b/devapp/devapp.go
index c24e24c..3bf7af0 100644
--- a/devapp/devapp.go
+++ b/devapp/devapp.go
@@ -43,11 +43,12 @@
 		autocertBucket = flag.String("autocert-bucket", "", "if non-empty, listen on port 443 and serve a LetsEncrypt TLS cert using this Google Cloud Storage bucket as a cache")
 		staticDir      = flag.String("static-dir", "./static/", "location of static directory relative to binary location")
 		templateDir    = flag.String("template-dir", "./templates/", "location of templates directory relative to binary location")
+		reload         = flag.Bool("reload", false, "reload content on each page load")
 	)
 	flag.Parse()
 	rand.Seed(time.Now().UnixNano())
 
-	s := newServer(http.NewServeMux(), *staticDir, *templateDir)
+	s := newServer(http.NewServeMux(), *staticDir, *templateDir, *reload)
 	ctx := context.Background()
 	if err := s.initCorpus(ctx); err != nil {
 		log.Fatalf("Could not init corpus: %v", err)
diff --git a/devapp/reviews.go b/devapp/reviews.go
index d00c252..7634c51 100644
--- a/devapp/reviews.go
+++ b/devapp/reviews.go
@@ -118,36 +118,11 @@
 	var (
 		projects     []*project
 		totalChanges int
-
-		excludedProjects = map[string]bool{
-			"gocloud":              true,
-			"google-api-go-client": true,
-		}
-		deletedChanges = map[struct {
-			proj string
-			num  int32
-		}]bool{
-			{"crypto", 35958}:  true,
-			{"scratch", 71730}: true,
-			{"scratch", 71850}: true,
-			{"scratch", 72090}: true,
-			{"scratch", 72091}: true,
-			{"scratch", 72110}: true,
-			{"scratch", 72131}: true,
-			{"tools", 93515}:   true,
-		}
 	)
-	s.corpus.Gerrit().ForeachProjectUnsorted(func(p *maintner.GerritProject) error {
-		if excludedProjects[p.Project()] {
-			return nil
-		}
+	s.corpus.Gerrit().ForeachProjectUnsorted(filterProjects(func(p *maintner.GerritProject) error {
 		proj := &project{GerritProject: p}
-		p.ForeachOpenCL(func(cl *maintner.GerritCL) error {
-			if deletedChanges[struct {
-				proj string
-				num  int32
-			}{p.Project(), cl.Number}] ||
-				cl.WorkInProgress() ||
+		p.ForeachOpenCL(withoutDeletedCLs(p, func(cl *maintner.GerritCL) error {
+			if cl.WorkInProgress() ||
 				cl.Owner() == nil ||
 				strings.Contains(cl.Commit.Msg, "DO NOT REVIEW") {
 				return nil
@@ -200,13 +175,13 @@
 			proj.Changes = append(proj.Changes, c)
 			totalChanges++
 			return nil
-		})
+		}))
 		sort.Slice(proj.Changes, func(i, j int) bool {
 			return proj.Changes[i].LastUpdate.Before(proj.Changes[j].LastUpdate)
 		})
 		projects = append(projects, proj)
 		return nil
-	})
+	}))
 	sort.Slice(projects, func(i, j int) bool {
 		return projects[i].Project() < projects[j].Project()
 	})
diff --git a/devapp/server.go b/devapp/server.go
index 9a86cd9..3a47ec6 100644
--- a/devapp/server.go
+++ b/devapp/server.go
@@ -29,6 +29,7 @@
 	mux         *http.ServeMux
 	staticDir   string
 	templateDir string
+	reloadTmpls bool
 
 	cMu              sync.RWMutex // Used to protect the fields below.
 	corpus           *maintner.Corpus
@@ -50,13 +51,15 @@
 type pageData struct {
 	release releaseData
 	reviews reviewsData
+	stats   statsData
 }
 
-func newServer(mux *http.ServeMux, staticDir, templateDir string) *server {
+func newServer(mux *http.ServeMux, staticDir, templateDir string, reloadTmpls bool) *server {
 	s := &server{
 		mux:         mux,
 		staticDir:   staticDir,
 		templateDir: templateDir,
+		reloadTmpls: reloadTmpls,
 		userMapping: map[int]*maintner.GitHubUser{},
 	}
 	s.mux.Handle("/", http.FileServer(http.Dir(s.staticDir)))
@@ -64,6 +67,7 @@
 	s.mux.HandleFunc("/favicon.ico", s.handleFavicon)
 	s.mux.HandleFunc("/release", s.withTemplate("/release.tmpl", s.handleRelease))
 	s.mux.HandleFunc("/reviews", s.withTemplate("/reviews.tmpl", s.handleReviews))
+	s.mux.HandleFunc("/stats", s.withTemplate("/stats.tmpl", s.handleStats))
 	s.mux.HandleFunc("/dir/", handleDirRedirect)
 	s.mux.HandleFunc("/owners", owners.Handler)
 	s.mux.Handle("/owners/", http.RedirectHandler("/owners", http.StatusPermanentRedirect)) // TODO: remove after clients updated to use URL without trailing slash
@@ -76,7 +80,12 @@
 
 func (s *server) withTemplate(tmpl string, fn func(*template.Template, http.ResponseWriter, *http.Request)) http.HandlerFunc {
 	t := template.Must(template.ParseFiles(path.Join(s.templateDir, tmpl)))
-	return func(w http.ResponseWriter, r *http.Request) { fn(t, w, r) }
+	return func(w http.ResponseWriter, r *http.Request) {
+		if s.reloadTmpls {
+			t = template.Must(template.ParseFiles(path.Join(s.templateDir, tmpl)))
+		}
+		fn(t, w, r)
+	}
 }
 
 // initCorpus fetches a full maintner corpus, overwriting any existing data.
@@ -107,6 +116,7 @@
 		s.cMu.Lock()
 		s.data.release.dirty = true
 		s.data.reviews.dirty = true
+		s.data.stats.dirty = true
 		s.cMu.Unlock()
 		err := s.corpus.UpdateWithLocker(ctx, &s.cMu)
 		if err != nil {
diff --git a/devapp/server_test.go b/devapp/server_test.go
index 8056ec2..c03794b 100644
--- a/devapp/server_test.go
+++ b/devapp/server_test.go
@@ -11,7 +11,7 @@
 	"testing"
 )
 
-var testServer = newServer(http.DefaultServeMux, "./static/", "./templates/")
+var testServer = newServer(http.DefaultServeMux, "./static/", "./templates/", false)
 
 func TestStaticAssetsFound(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", nil)
diff --git a/devapp/static/index.html b/devapp/static/index.html
index 34bddd8..ac70004 100644
--- a/devapp/static/index.html
+++ b/devapp/static/index.html
@@ -1,10 +1,13 @@
 <!DOCTYPE html>
+<html lang="en">
 <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Go Development Dashboard</title>
 <pre>
 <a href="/release">Releases</a>
 <a href="/reviews">Open reviews</a>
 <a href="/owners">Owners</a>
+<a href="/stats">Stats</a>
 
 <b>About the Dashboards</b>
 
diff --git a/devapp/stats.go b/devapp/stats.go
new file mode 100644
index 0000000..713a0af
--- /dev/null
+++ b/devapp/stats.go
@@ -0,0 +1,181 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"html/template"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/build/maintner"
+)
+
+// handleStats serves dev.golang.org/stats.
+func (s *server) handleStats(t *template.Template, w http.ResponseWriter, r *http.Request) {
+	s.cMu.RLock()
+	dirty := s.data.stats.dirty
+	s.cMu.RUnlock()
+	if dirty {
+		s.updateStatsData()
+	}
+
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	var buf bytes.Buffer
+	s.cMu.RLock()
+	defer s.cMu.RUnlock()
+	data := struct {
+		DataJSON interface{}
+	}{
+		DataJSON: s.data.stats,
+	}
+	if err := t.Execute(&buf, data); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if _, err := io.Copy(w, &buf); err != nil {
+		log.Printf("io.Copy(w, %+v) = %v", buf, err)
+		return
+	}
+}
+
+type statsData struct {
+	Charts []*chart
+
+	// dirty is set if this data needs to be updated due to a corpus change.
+	dirty bool
+}
+
+// A chart holds data used by the Google Charts JavaScript API to render
+// an interactive visualization.
+type chart struct {
+	Title   string          `json:"title"`
+	Columns []*chartColumn  `json:"columns"`
+	Data    [][]interface{} `json:"data"`
+}
+
+// A chartColumn is analogous to a Google Charts DataTable column.
+type chartColumn struct {
+	// Type is the data type of the values of the column.
+	// Supported values are 'string', 'number', 'boolean',
+	// 'timeofday', 'date', and 'datetime'.
+	Type string `json:"type"`
+
+	// Label is an optional label for the column.
+	Label string `json:"label"`
+}
+
+func (s *server) updateStatsData() {
+	log.Println("Updating stats data ...")
+	s.cMu.Lock()
+	defer s.cMu.Unlock()
+
+	var (
+		windowStart = time.Now().Add(-1 * 365 * 24 * time.Hour)
+		intervals   []*clInterval
+	)
+	s.corpus.Gerrit().ForeachProjectUnsorted(filterProjects(func(p *maintner.GerritProject) error {
+		p.ForeachCLUnsorted(withoutDeletedCLs(p, func(cl *maintner.GerritCL) error {
+			closed := cl.Status == "merged" || cl.Status == "abandoned"
+
+			// Discard CL if closed and last updated before windowStart.
+			if closed && cl.Meta.Commit.CommitTime.Before(windowStart) {
+				return nil
+			}
+			intervals = append(intervals, newIntervalFromCL(cl))
+			return nil
+		}))
+		return nil
+	}))
+
+	var chartData [][]interface{}
+	for t0, t1 := windowStart, windowStart.Add(24*time.Hour); t0.Before(time.Now()); t0, t1 = t0.Add(24*time.Hour), t1.Add(24*time.Hour) {
+		var (
+			open       int
+			withIssues int
+		)
+
+		for _, i := range intervals {
+			if !i.intersects(t0, t1) {
+				continue
+			}
+			open++
+			if len(i.cl.GitHubIssueRefs) > 0 {
+				withIssues++
+			}
+		}
+		chartData = append(chartData, []interface{}{
+			t0, open, withIssues,
+		})
+	}
+	cols := []*chartColumn{
+		{Type: "date", Label: "date"},
+		{Type: "number", Label: "All CLs"},
+		{Type: "number", Label: "With issues"},
+	}
+	var charts []*chart
+	charts = append(charts, &chart{
+		Title:   "Open CLs (1 Year)",
+		Columns: cols,
+		Data:    chartData,
+	})
+	charts = append(charts, &chart{
+		Title:   "Open CLs (30 Days)",
+		Columns: cols,
+		Data:    chartData[len(chartData)-30:],
+	})
+	charts = append(charts, &chart{
+		Title:   "Open CLs (7 Days)",
+		Columns: cols,
+		Data:    chartData[len(chartData)-7:],
+	})
+	s.data.stats.Charts = charts
+}
+
+// A clInterval describes a time period during which a CL is open.
+// points on the interval are seconds since the epoch.
+type clInterval struct {
+	start, end int64 // seconds since epoch
+	cl         *maintner.GerritCL
+}
+
+// returns true iff the interval contains any seconds
+// in the timespan [t0,t1]. t0 must be before t1.
+func (i *clInterval) intersects(t0, t1 time.Time) bool {
+	if t1.Before(t0) {
+		panic("t0 cannot be before t1")
+	}
+	return i.end >= t0.Unix() && i.start <= t1.Unix()
+}
+
+func newIntervalFromCL(cl *maintner.GerritCL) *clInterval {
+	interval := &clInterval{
+		start: cl.Created.Unix(),
+		end:   math.MaxInt64,
+		cl:    cl,
+	}
+
+	closed := cl.Status == "merged" || cl.Status == "abandoned"
+	if closed {
+		for i := len(cl.Metas) - 1; i >= 0; i-- {
+			if !strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit") {
+				continue
+			}
+
+			if strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit:merged") ||
+				strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit:abandon") {
+				interval.end = cl.Metas[i].Commit.CommitTime.Unix()
+			}
+		}
+		if interval.end == math.MaxInt64 {
+			log.Printf("Unable to determine close time of CL: %+v", cl)
+		}
+	}
+	return interval
+}
diff --git a/devapp/stats_test.go b/devapp/stats_test.go
new file mode 100644
index 0000000..c15d996
--- /dev/null
+++ b/devapp/stats_test.go
@@ -0,0 +1,117 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"golang.org/x/build/maintner"
+)
+
+func TestNewIntervalFromCL(t *testing.T) {
+	var (
+		t0 = time.Now()
+		t1 = t0.Add(1 * time.Hour)
+	)
+	testCases := []struct {
+		cl         *maintner.GerritCL
+		start, end int64
+	}{
+		{
+			cl: &maintner.GerritCL{
+				Created: t0,
+				Status:  "new",
+			},
+			start: t0.Unix(),
+			end:   math.MaxInt64,
+		},
+		{
+			cl: &maintner.GerritCL{
+				Created: t0,
+				Status:  "merged",
+				Metas: []*maintner.GerritMeta{
+					{
+						Commit: &maintner.GitCommit{
+							Msg:        "autogenerated:gerrit:merged",
+							CommitTime: t1,
+						},
+					},
+				},
+			},
+			start: t0.Unix(),
+			end:   t1.Unix(),
+		},
+		{
+			cl: &maintner.GerritCL{
+				Created: t0,
+				Status:  "abandoned",
+				Metas: []*maintner.GerritMeta{
+					{
+						Commit: &maintner.GitCommit{
+							Msg:        "autogenerated:gerrit:abandon",
+							CommitTime: t1,
+						},
+					},
+				},
+			},
+			start: t0.Unix(),
+			end:   t1.Unix(),
+		},
+	}
+
+	for _, tc := range testCases {
+		ival := newIntervalFromCL(tc.cl)
+		if got, want := ival.start, tc.start; got != want {
+			t.Errorf("start: got %d; want %d", got, want)
+		}
+		if got, want := ival.end, tc.end; got != want {
+			t.Errorf("end: got %d; want %d", got, want)
+		}
+		if got, want := ival.cl, tc.cl; got != want {
+			t.Errorf("cl: got %+v; want %+v", got, want)
+		}
+	}
+}
+
+func TestIntervalIntersection(t *testing.T) {
+	testCases := []struct {
+		interval   *clInterval
+		t0, t1     time.Time
+		intersects bool
+	}{
+		{
+			&clInterval{start: 0, end: 5},
+			time.Unix(0, 0),
+			time.Unix(10, 0),
+			true,
+		},
+		{
+			&clInterval{start: 10, end: 20},
+			time.Unix(0, 0),
+			time.Unix(10, 0),
+			true,
+		},
+		{
+			&clInterval{start: 10, end: 20},
+			time.Unix(0, 0),
+			time.Unix(9, 0),
+			false,
+		},
+		{
+			&clInterval{start: 0, end: 5},
+			time.Unix(6, 0),
+			time.Unix(10, 0),
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		if got, want := tc.interval.intersects(tc.t0, tc.t1), tc.intersects; got != want {
+			t.Errorf("(%v).intersects(%v, %v): got %v; want %v", tc.interval, tc.t0, tc.t1, got, want)
+		}
+	}
+}
diff --git a/devapp/templates/release.tmpl b/devapp/templates/release.tmpl
index 7cfc02b..952a54b 100644
--- a/devapp/templates/release.tmpl
+++ b/devapp/templates/release.tmpl
@@ -1,7 +1,8 @@
 <!DOCTYPE html>
+<html lang="en">
 <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Go Release Dashboard</title>
-<meta name=viewport content="width=device-width,minimum-scale=1,maximum-scale=1">
 <style>
 * {
   box-sizing: border-box;
diff --git a/devapp/templates/reviews.tmpl b/devapp/templates/reviews.tmpl
index 75ff29b..73fe8af 100644
--- a/devapp/templates/reviews.tmpl
+++ b/devapp/templates/reviews.tmpl
@@ -1,6 +1,7 @@
 <!DOCTYPE html>
+<html lang="en">
 <meta charset="utf-8">
-<meta name=viewport content="width=device-width,minimum-scale=1,maximum-scale=1">
+<meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Open Go Code Reviews</title>
 <style>
 * {
diff --git a/devapp/templates/stats.tmpl b/devapp/templates/stats.tmpl
new file mode 100644
index 0000000..9f9f8b6
--- /dev/null
+++ b/devapp/templates/stats.tmpl
@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link rel="preconnect" href="https://www.gstatic.com">
+<title>Go Stats</title>
+<style>
+* {
+  box-sizing: border-box;
+  margin: 0;
+  padding: 0;
+}
+body {
+  font: 13px system-ui, sans-serif;
+  padding: 1rem;
+}
+.Chart {
+  border: 1px solid #999;
+  height: 400px;
+  padding: 1rem;
+}
+.Chart + .Chart {
+  margin-top: 1rem;
+}
+@media only screen and (min-device-width: 37.5rem) {
+  .ChartsContainer {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(37.5rem, 1fr));
+    grid-gap: 1rem;
+  }
+  .Chart + .Chart {
+    margin-top: 0;
+  }
+}
+</style>
+<header>
+  <h1>Go Stats</h1>
+</header>
+<main>
+  <div class="ChartsContainer js-chartContainer"></div>
+</main>
+<script src="https://www.gstatic.com/charts/loader.js"></script>
+<script>
+  const CHART_DATA = {{.DataJSON}};
+  for (let chart of CHART_DATA.Charts) {
+    const dateColumns = new Set(); // index -> date type
+    for (let i = 0; i < chart.columns.length; i++) {
+      if (chart.columns[i].type === 'date' || chart.columns[i].type === 'datetime') {
+        dateColumns.add(i);
+      }
+    }
+    if (dateColumns.size > 0) {
+      for (let row = 0; row < chart.data.length; row++) {
+        for (let col of dateColumns) {
+          chart.data[row][col] = new Date(chart.data[row][col]);
+        }
+      }
+    }
+  }
+  const data = []; // indexed by charts entry
+  const googCharts = [];  // indexed by charts entry
+
+  google.charts.load('current', {packages:['corechart']});
+  google.charts.setOnLoadCallback(() => {
+    for (let i = 0; i < CHART_DATA.Charts.length; i++) {
+      const chart = CHART_DATA.Charts[i];
+      data[i] = new google.visualization.DataTable();
+      for (let col of chart.columns) {
+        data[i].addColumn(col.type, col.label);
+      }
+      data[i].addRows(chart.data);
+      const containerEl = document.createElement('div');
+      containerEl.classList.add('Chart');
+      document.querySelector('.js-chartContainer').appendChild(containerEl);
+      googCharts[i] = new google.visualization.LineChart(containerEl);
+    }
+    drawCharts();
+  });
+
+  const drawCharts = () => {
+    for (let i = 0; i < CHART_DATA.Charts.length; i++) {
+      googCharts[i].draw(data[i], {
+        title: CHART_DATA.Charts[i].title,
+        chartArea: {width: '100%', height: '80%'},
+        legend: {position: 'top', alignment: 'end'},
+      });
+    }
+  };
+
+  let debounceTimerId;
+  window.addEventListener('resize', e => {
+    if (debounceTimerId != null) {
+      window.clearTimeout(debounceTimerId);
+      debounceTimerId = null;
+    }
+    debounceTimerId = window.setTimeout(() => { drawCharts(); }, 50);
+  });
+</script>