devapp: add stats page plus some minor cleanup
+ Adds a stats page that shows open CLs over time
+ Adds a command-line flag to reload templates on each request
+ Breaks some repeated code into some shared logic
+ Some minor cleanup to adhere to best practices
Change-Id: I2d84a1e5c77c7e9131c758d69e4a9bf9a9d815f4
Reviewed-on: https://go-review.googlesource.com/c/build/+/199637
Run-TryBot: Andrew Bonventre <andybons@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alexander Rakoczy <alex@golang.org>
diff --git a/devapp/data.go b/devapp/data.go
new file mode 100644
index 0000000..28ccbba
--- /dev/null
+++ b/devapp/data.go
@@ -0,0 +1,50 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "golang.org/x/build/maintner"
+)
+
+var (
+ excludedProjects = map[string]bool{
+ "gocloud": true,
+ "google-api-go-client": true,
+ }
+ deletedChanges = map[struct {
+ proj string
+ num int32
+ }]bool{
+ {"crypto", 35958}: true,
+ {"scratch", 71730}: true,
+ {"scratch", 71850}: true,
+ {"scratch", 72090}: true,
+ {"scratch", 72091}: true,
+ {"scratch", 72110}: true,
+ {"scratch", 72131}: true,
+ {"tools", 93515}: true,
+ }
+)
+
+func filterProjects(fn func(*maintner.GerritProject) error) func(*maintner.GerritProject) error {
+ return func(p *maintner.GerritProject) error {
+ if excludedProjects[p.Project()] {
+ return nil
+ }
+ return fn(p)
+ }
+}
+
+func withoutDeletedCLs(p *maintner.GerritProject, fn func(*maintner.GerritCL) error) func(*maintner.GerritCL) error {
+ return func(cl *maintner.GerritCL) error {
+ if deletedChanges[struct {
+ proj string
+ num int32
+ }{p.Project(), cl.Number}] {
+ return nil
+ }
+ return fn(cl)
+ }
+}
diff --git a/devapp/devapp.go b/devapp/devapp.go
index c24e24c..3bf7af0 100644
--- a/devapp/devapp.go
+++ b/devapp/devapp.go
@@ -43,11 +43,12 @@
autocertBucket = flag.String("autocert-bucket", "", "if non-empty, listen on port 443 and serve a LetsEncrypt TLS cert using this Google Cloud Storage bucket as a cache")
staticDir = flag.String("static-dir", "./static/", "location of static directory relative to binary location")
templateDir = flag.String("template-dir", "./templates/", "location of templates directory relative to binary location")
+ reload = flag.Bool("reload", false, "reload content on each page load")
)
flag.Parse()
rand.Seed(time.Now().UnixNano())
- s := newServer(http.NewServeMux(), *staticDir, *templateDir)
+ s := newServer(http.NewServeMux(), *staticDir, *templateDir, *reload)
ctx := context.Background()
if err := s.initCorpus(ctx); err != nil {
log.Fatalf("Could not init corpus: %v", err)
diff --git a/devapp/reviews.go b/devapp/reviews.go
index d00c252..7634c51 100644
--- a/devapp/reviews.go
+++ b/devapp/reviews.go
@@ -118,36 +118,11 @@
var (
projects []*project
totalChanges int
-
- excludedProjects = map[string]bool{
- "gocloud": true,
- "google-api-go-client": true,
- }
- deletedChanges = map[struct {
- proj string
- num int32
- }]bool{
- {"crypto", 35958}: true,
- {"scratch", 71730}: true,
- {"scratch", 71850}: true,
- {"scratch", 72090}: true,
- {"scratch", 72091}: true,
- {"scratch", 72110}: true,
- {"scratch", 72131}: true,
- {"tools", 93515}: true,
- }
)
- s.corpus.Gerrit().ForeachProjectUnsorted(func(p *maintner.GerritProject) error {
- if excludedProjects[p.Project()] {
- return nil
- }
+ s.corpus.Gerrit().ForeachProjectUnsorted(filterProjects(func(p *maintner.GerritProject) error {
proj := &project{GerritProject: p}
- p.ForeachOpenCL(func(cl *maintner.GerritCL) error {
- if deletedChanges[struct {
- proj string
- num int32
- }{p.Project(), cl.Number}] ||
- cl.WorkInProgress() ||
+ p.ForeachOpenCL(withoutDeletedCLs(p, func(cl *maintner.GerritCL) error {
+ if cl.WorkInProgress() ||
cl.Owner() == nil ||
strings.Contains(cl.Commit.Msg, "DO NOT REVIEW") {
return nil
@@ -200,13 +175,13 @@
proj.Changes = append(proj.Changes, c)
totalChanges++
return nil
- })
+ }))
sort.Slice(proj.Changes, func(i, j int) bool {
return proj.Changes[i].LastUpdate.Before(proj.Changes[j].LastUpdate)
})
projects = append(projects, proj)
return nil
- })
+ }))
sort.Slice(projects, func(i, j int) bool {
return projects[i].Project() < projects[j].Project()
})
diff --git a/devapp/server.go b/devapp/server.go
index 9a86cd9..3a47ec6 100644
--- a/devapp/server.go
+++ b/devapp/server.go
@@ -29,6 +29,7 @@
mux *http.ServeMux
staticDir string
templateDir string
+ reloadTmpls bool
cMu sync.RWMutex // Used to protect the fields below.
corpus *maintner.Corpus
@@ -50,13 +51,15 @@
type pageData struct {
release releaseData
reviews reviewsData
+ stats statsData
}
-func newServer(mux *http.ServeMux, staticDir, templateDir string) *server {
+func newServer(mux *http.ServeMux, staticDir, templateDir string, reloadTmpls bool) *server {
s := &server{
mux: mux,
staticDir: staticDir,
templateDir: templateDir,
+ reloadTmpls: reloadTmpls,
userMapping: map[int]*maintner.GitHubUser{},
}
s.mux.Handle("/", http.FileServer(http.Dir(s.staticDir)))
@@ -64,6 +67,7 @@
s.mux.HandleFunc("/favicon.ico", s.handleFavicon)
s.mux.HandleFunc("/release", s.withTemplate("/release.tmpl", s.handleRelease))
s.mux.HandleFunc("/reviews", s.withTemplate("/reviews.tmpl", s.handleReviews))
+ s.mux.HandleFunc("/stats", s.withTemplate("/stats.tmpl", s.handleStats))
s.mux.HandleFunc("/dir/", handleDirRedirect)
s.mux.HandleFunc("/owners", owners.Handler)
s.mux.Handle("/owners/", http.RedirectHandler("/owners", http.StatusPermanentRedirect)) // TODO: remove after clients updated to use URL without trailing slash
@@ -76,7 +80,12 @@
func (s *server) withTemplate(tmpl string, fn func(*template.Template, http.ResponseWriter, *http.Request)) http.HandlerFunc {
t := template.Must(template.ParseFiles(path.Join(s.templateDir, tmpl)))
- return func(w http.ResponseWriter, r *http.Request) { fn(t, w, r) }
+ return func(w http.ResponseWriter, r *http.Request) {
+ if s.reloadTmpls {
+ t = template.Must(template.ParseFiles(path.Join(s.templateDir, tmpl)))
+ }
+ fn(t, w, r)
+ }
}
// initCorpus fetches a full maintner corpus, overwriting any existing data.
@@ -107,6 +116,7 @@
s.cMu.Lock()
s.data.release.dirty = true
s.data.reviews.dirty = true
+ s.data.stats.dirty = true
s.cMu.Unlock()
err := s.corpus.UpdateWithLocker(ctx, &s.cMu)
if err != nil {
diff --git a/devapp/server_test.go b/devapp/server_test.go
index 8056ec2..c03794b 100644
--- a/devapp/server_test.go
+++ b/devapp/server_test.go
@@ -11,7 +11,7 @@
"testing"
)
-var testServer = newServer(http.DefaultServeMux, "./static/", "./templates/")
+var testServer = newServer(http.DefaultServeMux, "./static/", "./templates/", false)
func TestStaticAssetsFound(t *testing.T) {
req := httptest.NewRequest("GET", "/", nil)
diff --git a/devapp/static/index.html b/devapp/static/index.html
index 34bddd8..ac70004 100644
--- a/devapp/static/index.html
+++ b/devapp/static/index.html
@@ -1,10 +1,13 @@
<!DOCTYPE html>
+<html lang="en">
<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Go Development Dashboard</title>
<pre>
<a href="/release">Releases</a>
<a href="/reviews">Open reviews</a>
<a href="/owners">Owners</a>
+<a href="/stats">Stats</a>
<b>About the Dashboards</b>
diff --git a/devapp/stats.go b/devapp/stats.go
new file mode 100644
index 0000000..713a0af
--- /dev/null
+++ b/devapp/stats.go
@@ -0,0 +1,181 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "html/template"
+ "io"
+ "log"
+ "math"
+ "net/http"
+ "strings"
+ "time"
+
+ "golang.org/x/build/maintner"
+)
+
+// handleStats serves dev.golang.org/stats.
+func (s *server) handleStats(t *template.Template, w http.ResponseWriter, r *http.Request) {
+ s.cMu.RLock()
+ dirty := s.data.stats.dirty
+ s.cMu.RUnlock()
+ if dirty {
+ s.updateStatsData()
+ }
+
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ var buf bytes.Buffer
+ s.cMu.RLock()
+ defer s.cMu.RUnlock()
+ data := struct {
+ DataJSON interface{}
+ }{
+ DataJSON: s.data.stats,
+ }
+ if err := t.Execute(&buf, data); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if _, err := io.Copy(w, &buf); err != nil {
+ log.Printf("io.Copy(w, %+v) = %v", buf, err)
+ return
+ }
+}
+
+type statsData struct {
+ Charts []*chart
+
+ // dirty is set if this data needs to be updated due to a corpus change.
+ dirty bool
+}
+
+// A chart holds data used by the Google Charts JavaScript API to render
+// an interactive visualization.
+type chart struct {
+ Title string `json:"title"`
+ Columns []*chartColumn `json:"columns"`
+ Data [][]interface{} `json:"data"`
+}
+
+// A chartColumn is analogous to a Google Charts DataTable column.
+type chartColumn struct {
+ // Type is the data type of the values of the column.
+ // Supported values are 'string', 'number', 'boolean',
+ // 'timeofday', 'date', and 'datetime'.
+ Type string `json:"type"`
+
+ // Label is an optional label for the column.
+ Label string `json:"label"`
+}
+
+func (s *server) updateStatsData() {
+ log.Println("Updating stats data ...")
+ s.cMu.Lock()
+ defer s.cMu.Unlock()
+
+ var (
+ windowStart = time.Now().Add(-1 * 365 * 24 * time.Hour)
+ intervals []*clInterval
+ )
+ s.corpus.Gerrit().ForeachProjectUnsorted(filterProjects(func(p *maintner.GerritProject) error {
+ p.ForeachCLUnsorted(withoutDeletedCLs(p, func(cl *maintner.GerritCL) error {
+ closed := cl.Status == "merged" || cl.Status == "abandoned"
+
+ // Discard CL if closed and last updated before windowStart.
+ if closed && cl.Meta.Commit.CommitTime.Before(windowStart) {
+ return nil
+ }
+ intervals = append(intervals, newIntervalFromCL(cl))
+ return nil
+ }))
+ return nil
+ }))
+
+ var chartData [][]interface{}
+ for t0, t1 := windowStart, windowStart.Add(24*time.Hour); t0.Before(time.Now()); t0, t1 = t0.Add(24*time.Hour), t1.Add(24*time.Hour) {
+ var (
+ open int
+ withIssues int
+ )
+
+ for _, i := range intervals {
+ if !i.intersects(t0, t1) {
+ continue
+ }
+ open++
+ if len(i.cl.GitHubIssueRefs) > 0 {
+ withIssues++
+ }
+ }
+ chartData = append(chartData, []interface{}{
+ t0, open, withIssues,
+ })
+ }
+ cols := []*chartColumn{
+ {Type: "date", Label: "date"},
+ {Type: "number", Label: "All CLs"},
+ {Type: "number", Label: "With issues"},
+ }
+ var charts []*chart
+ charts = append(charts, &chart{
+ Title: "Open CLs (1 Year)",
+ Columns: cols,
+ Data: chartData,
+ })
+ charts = append(charts, &chart{
+ Title: "Open CLs (30 Days)",
+ Columns: cols,
+ Data: chartData[len(chartData)-30:],
+ })
+ charts = append(charts, &chart{
+ Title: "Open CLs (7 Days)",
+ Columns: cols,
+ Data: chartData[len(chartData)-7:],
+ })
+ s.data.stats.Charts = charts
+}
+
+// A clInterval describes a time period during which a CL is open.
+// points on the interval are seconds since the epoch.
+type clInterval struct {
+ start, end int64 // seconds since epoch
+ cl *maintner.GerritCL
+}
+
+// returns true iff the interval contains any seconds
+// in the timespan [t0,t1]. t0 must be before t1.
+func (i *clInterval) intersects(t0, t1 time.Time) bool {
+ if t1.Before(t0) {
+ panic("t0 cannot be before t1")
+ }
+ return i.end >= t0.Unix() && i.start <= t1.Unix()
+}
+
+func newIntervalFromCL(cl *maintner.GerritCL) *clInterval {
+ interval := &clInterval{
+ start: cl.Created.Unix(),
+ end: math.MaxInt64,
+ cl: cl,
+ }
+
+ closed := cl.Status == "merged" || cl.Status == "abandoned"
+ if closed {
+ for i := len(cl.Metas) - 1; i >= 0; i-- {
+ if !strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit") {
+ continue
+ }
+
+ if strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit:merged") ||
+ strings.Contains(cl.Metas[i].Commit.Msg, "autogenerated:gerrit:abandon") {
+ interval.end = cl.Metas[i].Commit.CommitTime.Unix()
+ }
+ }
+ if interval.end == math.MaxInt64 {
+ log.Printf("Unable to determine close time of CL: %+v", cl)
+ }
+ }
+ return interval
+}
diff --git a/devapp/stats_test.go b/devapp/stats_test.go
new file mode 100644
index 0000000..c15d996
--- /dev/null
+++ b/devapp/stats_test.go
@@ -0,0 +1,117 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "math"
+ "testing"
+ "time"
+
+ "golang.org/x/build/maintner"
+)
+
+func TestNewIntervalFromCL(t *testing.T) {
+ var (
+ t0 = time.Now()
+ t1 = t0.Add(1 * time.Hour)
+ )
+ testCases := []struct {
+ cl *maintner.GerritCL
+ start, end int64
+ }{
+ {
+ cl: &maintner.GerritCL{
+ Created: t0,
+ Status: "new",
+ },
+ start: t0.Unix(),
+ end: math.MaxInt64,
+ },
+ {
+ cl: &maintner.GerritCL{
+ Created: t0,
+ Status: "merged",
+ Metas: []*maintner.GerritMeta{
+ {
+ Commit: &maintner.GitCommit{
+ Msg: "autogenerated:gerrit:merged",
+ CommitTime: t1,
+ },
+ },
+ },
+ },
+ start: t0.Unix(),
+ end: t1.Unix(),
+ },
+ {
+ cl: &maintner.GerritCL{
+ Created: t0,
+ Status: "abandoned",
+ Metas: []*maintner.GerritMeta{
+ {
+ Commit: &maintner.GitCommit{
+ Msg: "autogenerated:gerrit:abandon",
+ CommitTime: t1,
+ },
+ },
+ },
+ },
+ start: t0.Unix(),
+ end: t1.Unix(),
+ },
+ }
+
+ for _, tc := range testCases {
+ ival := newIntervalFromCL(tc.cl)
+ if got, want := ival.start, tc.start; got != want {
+ t.Errorf("start: got %d; want %d", got, want)
+ }
+ if got, want := ival.end, tc.end; got != want {
+ t.Errorf("end: got %d; want %d", got, want)
+ }
+ if got, want := ival.cl, tc.cl; got != want {
+ t.Errorf("cl: got %+v; want %+v", got, want)
+ }
+ }
+}
+
+func TestIntervalIntersection(t *testing.T) {
+ testCases := []struct {
+ interval *clInterval
+ t0, t1 time.Time
+ intersects bool
+ }{
+ {
+ &clInterval{start: 0, end: 5},
+ time.Unix(0, 0),
+ time.Unix(10, 0),
+ true,
+ },
+ {
+ &clInterval{start: 10, end: 20},
+ time.Unix(0, 0),
+ time.Unix(10, 0),
+ true,
+ },
+ {
+ &clInterval{start: 10, end: 20},
+ time.Unix(0, 0),
+ time.Unix(9, 0),
+ false,
+ },
+ {
+ &clInterval{start: 0, end: 5},
+ time.Unix(6, 0),
+ time.Unix(10, 0),
+ false,
+ },
+ }
+
+ for _, tc := range testCases {
+ if got, want := tc.interval.intersects(tc.t0, tc.t1), tc.intersects; got != want {
+ t.Errorf("(%v).intersects(%v, %v): got %v; want %v", tc.interval, tc.t0, tc.t1, got, want)
+ }
+ }
+}
diff --git a/devapp/templates/release.tmpl b/devapp/templates/release.tmpl
index 7cfc02b..952a54b 100644
--- a/devapp/templates/release.tmpl
+++ b/devapp/templates/release.tmpl
@@ -1,7 +1,8 @@
<!DOCTYPE html>
+<html lang="en">
<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Go Release Dashboard</title>
-<meta name=viewport content="width=device-width,minimum-scale=1,maximum-scale=1">
<style>
* {
box-sizing: border-box;
diff --git a/devapp/templates/reviews.tmpl b/devapp/templates/reviews.tmpl
index 75ff29b..73fe8af 100644
--- a/devapp/templates/reviews.tmpl
+++ b/devapp/templates/reviews.tmpl
@@ -1,6 +1,7 @@
<!DOCTYPE html>
+<html lang="en">
<meta charset="utf-8">
-<meta name=viewport content="width=device-width,minimum-scale=1,maximum-scale=1">
+<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Open Go Code Reviews</title>
<style>
* {
diff --git a/devapp/templates/stats.tmpl b/devapp/templates/stats.tmpl
new file mode 100644
index 0000000..9f9f8b6
--- /dev/null
+++ b/devapp/templates/stats.tmpl
@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link rel="preconnect" href="https://www.gstatic.com">
+<title>Go Stats</title>
+<style>
+* {
+ box-sizing: border-box;
+ margin: 0;
+ padding: 0;
+}
+body {
+ font: 13px system-ui, sans-serif;
+ padding: 1rem;
+}
+.Chart {
+ border: 1px solid #999;
+ height: 400px;
+ padding: 1rem;
+}
+.Chart + .Chart {
+ margin-top: 1rem;
+}
+@media only screen and (min-device-width: 37.5rem) {
+ .ChartsContainer {
+ display: grid;
+ grid-template-columns: repeat(auto-fill, minmax(37.5rem, 1fr));
+ grid-gap: 1rem;
+ }
+ .Chart + .Chart {
+ margin-top: 0;
+ }
+}
+</style>
+<header>
+ <h1>Go Stats</h1>
+</header>
+<main>
+ <div class="ChartsContainer js-chartContainer"></div>
+</main>
+<script src="https://www.gstatic.com/charts/loader.js"></script>
+<script>
+ const CHART_DATA = {{.DataJSON}};
+ for (let chart of CHART_DATA.Charts) {
+ const dateColumns = new Set(); // index -> date type
+ for (let i = 0; i < chart.columns.length; i++) {
+ if (chart.columns[i].type === 'date' || chart.columns[i].type === 'datetime') {
+ dateColumns.add(i);
+ }
+ }
+ if (dateColumns.size > 0) {
+ for (let row = 0; row < chart.data.length; row++) {
+ for (let col of dateColumns) {
+ chart.data[row][col] = new Date(chart.data[row][col]);
+ }
+ }
+ }
+ }
+ const data = []; // indexed by charts entry
+ const googCharts = []; // indexed by charts entry
+
+ google.charts.load('current', {packages:['corechart']});
+ google.charts.setOnLoadCallback(() => {
+ for (let i = 0; i < CHART_DATA.Charts.length; i++) {
+ const chart = CHART_DATA.Charts[i];
+ data[i] = new google.visualization.DataTable();
+ for (let col of chart.columns) {
+ data[i].addColumn(col.type, col.label);
+ }
+ data[i].addRows(chart.data);
+ const containerEl = document.createElement('div');
+ containerEl.classList.add('Chart');
+ document.querySelector('.js-chartContainer').appendChild(containerEl);
+ googCharts[i] = new google.visualization.LineChart(containerEl);
+ }
+ drawCharts();
+ });
+
+ const drawCharts = () => {
+ for (let i = 0; i < CHART_DATA.Charts.length; i++) {
+ googCharts[i].draw(data[i], {
+ title: CHART_DATA.Charts[i].title,
+ chartArea: {width: '100%', height: '80%'},
+ legend: {position: 'top', alignment: 'end'},
+ });
+ }
+ };
+
+ let debounceTimerId;
+ window.addEventListener('resize', e => {
+ if (debounceTimerId != null) {
+ window.clearTimeout(debounceTimerId);
+ debounceTimerId = null;
+ }
+ debounceTimerId = window.setTimeout(() => { drawCharts(); }, 50);
+ });
+</script>