benchstat: split table generation into table.go, text.go, html.go
Change-Id: I33d4b6006f4df6b253133c4467488fd6da68a34f
Reviewed-on: https://go-review.googlesource.com/35937
Reviewed-by: Quentin Smith <quentin@golang.org>
diff --git a/cmd/benchstat/data.go b/cmd/benchstat/data.go
index c458b9f..c6ac4ec 100644
--- a/cmd/benchstat/data.go
+++ b/cmd/benchstat/data.go
@@ -59,7 +59,7 @@
// FormatDiff computes and formats the percent variation of max and min compared to mean.
// If b.Mean or b.Max is zero, FormatDiff returns an empty string.
func (m *Metrics) FormatDiff() string {
- if m.Mean == 0 || m.Max == 0 {
+ if m == nil || m.Mean == 0 || m.Max == 0 {
return ""
}
diff := 1 - m.Min/m.Mean
@@ -71,6 +71,9 @@
// Format returns a textual formatting of "Mean ±Diff" using scaler.
func (m *Metrics) Format(scaler Scaler) string {
+ if m == nil {
+ return ""
+ }
mean := m.FormatMean(scaler)
diff := m.FormatDiff()
if diff == "" {
diff --git a/cmd/benchstat/html.go b/cmd/benchstat/html.go
new file mode 100644
index 0000000..c4280ab
--- /dev/null
+++ b/cmd/benchstat/html.go
@@ -0,0 +1,39 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "html"
+)
+
+// FormatHTML appends an HTML formatting of the tables to buf.
+func FormatHTML(buf *bytes.Buffer, tables []*Table) {
+ var textTables [][]*textRow
+ for _, t := range tables {
+ textTables = append(textTables, toText(t))
+ }
+
+ for i, table := range textTables {
+ if i > 0 {
+ fmt.Fprintf(buf, "\n")
+ }
+ fmt.Fprintf(buf, "<style>.benchstat tbody td:nth-child(1n+2) { text-align: right; padding: 0em 1em; }</style>\n")
+ fmt.Fprintf(buf, "<table class='benchstat'>\n")
+ printRow := func(row *textRow, tag string) {
+ fmt.Fprintf(buf, "<tr>")
+ for _, cell := range row.cols {
+ fmt.Fprintf(buf, "<%s>%s</%s>", tag, html.EscapeString(cell), tag)
+ }
+ fmt.Fprintf(buf, "\n")
+ }
+ printRow(table[0], "th")
+ for _, row := range table[1:] {
+ printRow(row, "td")
+ }
+ fmt.Fprintf(buf, "</table>\n")
+ }
+}
diff --git a/cmd/benchstat/main.go b/cmd/benchstat/main.go
index 5813fd3..42c4c18 100644
--- a/cmd/benchstat/main.go
+++ b/cmd/benchstat/main.go
@@ -94,14 +94,10 @@
"bytes"
"flag"
"fmt"
- "html"
"io/ioutil"
"log"
"os"
"strings"
- "unicode/utf8"
-
- "golang.org/x/perf/internal/stats"
)
func usage() {
@@ -128,24 +124,6 @@
"ttest": TTest,
}
-type row struct {
- cols []string
-}
-
-func newRow(cols ...string) *row {
- return &row{cols: cols}
-}
-
-func (r *row) add(col string) {
- r.cols = append(r.cols, col)
-}
-
-func (r *row) trim() {
- for len(r.cols) > 0 && r.cols[len(r.cols)-1] == "" {
- r.cols = r.cols[:len(r.cols)-1]
- }
-}
-
func main() {
log.SetPrefix("benchstat: ")
log.SetFlags(0)
@@ -156,7 +134,6 @@
flag.Usage()
}
- // Read in benchmark data.
c := new(Collection)
for _, file := range flag.Args() {
data, err := ioutil.ReadFile(file)
@@ -165,210 +142,14 @@
}
c.AddConfig(file, data)
}
- for _, m := range c.Metrics {
- m.computeStats()
- }
- var tables [][]*row
- switch len(c.Configs) {
- case 2:
- before, after := c.Configs[0], c.Configs[1]
- key := Key{}
- for _, key.Unit = range c.Units {
- var table []*row
- metric := metricOf(key.Unit)
- for _, key.Benchmark = range c.Benchmarks {
- key.Config = before
- old := c.Metrics[key]
- key.Config = after
- new := c.Metrics[key]
- if old == nil || new == nil {
- continue
- }
- if len(table) == 0 {
- table = append(table, newRow("name", "old "+metric, "new "+metric, "delta"))
- }
-
- pval, testerr := deltaTest(old, new)
-
- scaler := NewScaler(old.Mean, old.Unit)
- row := newRow(key.Benchmark, old.Format(scaler), new.Format(scaler), "~ ")
- if testerr != nil {
- row.add(fmt.Sprintf("(%s)", testerr))
- } else if pval < *flagAlpha {
- row.cols[3] = fmt.Sprintf("%+.2f%%", ((new.Mean/old.Mean)-1.0)*100.0)
- }
- if len(row.cols) == 4 && pval != -1 {
- row.add(fmt.Sprintf("(p=%0.3f n=%d+%d)", pval, len(old.RValues), len(new.RValues)))
- }
- table = append(table, row)
- }
- if len(table) > 0 {
- table = addGeomean(table, c, key.Unit, true)
- tables = append(tables, table)
- }
- }
-
- default:
- key := Key{}
- for _, key.Unit = range c.Units {
- var table []*row
- metric := metricOf(key.Unit)
-
- if len(c.Configs) > 1 {
- hdr := newRow("name \\ " + metric)
- for _, config := range c.Configs {
- hdr.add(config)
- }
- table = append(table, hdr)
- } else {
- table = append(table, newRow("name", metric))
- }
-
- for _, key.Benchmark = range c.Benchmarks {
- row := newRow(key.Benchmark)
- var scaler Scaler
- for _, key.Config = range c.Configs {
- m := c.Metrics[key]
- if m == nil {
- row.add("")
- continue
- }
- if scaler == nil {
- scaler = NewScaler(m.Mean, m.Unit)
- }
- row.add(m.Format(scaler))
- }
- row.trim()
- if len(row.cols) > 1 {
- table = append(table, row)
- }
- }
- table = addGeomean(table, c, key.Unit, false)
- tables = append(tables, table)
- }
- }
-
- numColumn := 0
- for _, table := range tables {
- for _, row := range table {
- if numColumn < len(row.cols) {
- numColumn = len(row.cols)
- }
- }
- }
-
- max := make([]int, numColumn)
- for _, table := range tables {
- for _, row := range table {
- for i, s := range row.cols {
- n := utf8.RuneCountInString(s)
- if max[i] < n {
- max[i] = n
- }
- }
- }
- }
+ tables := c.Tables(deltaTest)
var buf bytes.Buffer
- for i, table := range tables {
- if i > 0 {
- fmt.Fprintf(&buf, "\n")
- }
-
- if *flagHTML {
- fmt.Fprintf(&buf, "<style>.benchstat tbody td:nth-child(1n+2) { text-align: right; padding: 0em 1em; }</style>\n")
- fmt.Fprintf(&buf, "<table class='benchstat'>\n")
- printRow := func(row *row, tag string) {
- fmt.Fprintf(&buf, "<tr>")
- for _, cell := range row.cols {
- fmt.Fprintf(&buf, "<%s>%s</%s>", tag, html.EscapeString(cell), tag)
- }
- fmt.Fprintf(&buf, "\n")
- }
- printRow(table[0], "th")
- for _, row := range table[1:] {
- printRow(row, "td")
- }
- fmt.Fprintf(&buf, "</table>\n")
- continue
- }
-
- // headings
- row := table[0]
- for i, s := range row.cols {
- switch i {
- case 0:
- fmt.Fprintf(&buf, "%-*s", max[i], s)
- default:
- fmt.Fprintf(&buf, " %-*s", max[i], s)
- case len(row.cols) - 1:
- fmt.Fprintf(&buf, " %s\n", s)
- }
- }
-
- // data
- for _, row := range table[1:] {
- for i, s := range row.cols {
- switch i {
- case 0:
- fmt.Fprintf(&buf, "%-*s", max[i], s)
- default:
- if i == len(row.cols)-1 && len(s) > 0 && s[0] == '(' {
- // Left-align p value.
- fmt.Fprintf(&buf, " %s", s)
- break
- }
- fmt.Fprintf(&buf, " %*s", max[i], s)
- }
- }
- fmt.Fprintf(&buf, "\n")
- }
+ if *flagHTML {
+ FormatHTML(&buf, tables)
+ } else {
+ FormatText(&buf, tables)
}
-
os.Stdout.Write(buf.Bytes())
}
-
-func addGeomean(table []*row, c *Collection, unit string, delta bool) []*row {
- if !*flagGeomean {
- return table
- }
-
- row := newRow("[Geo mean]")
- key := Key{Unit: unit}
- geomeans := []float64{}
- for _, key.Config = range c.Configs {
- var means []float64
- for _, key.Benchmark = range c.Benchmarks {
- m := c.Metrics[key]
- if m != nil {
- means = append(means, m.Mean)
- }
- }
- if len(means) == 0 {
- row.add("")
- delta = false
- } else {
- geomean := stats.GeoMean(means)
- geomeans = append(geomeans, geomean)
- row.add(NewScaler(geomean, unit)(geomean) + " ")
- }
- }
- if delta {
- row.add(fmt.Sprintf("%+.2f%%", ((geomeans[1]/geomeans[0])-1.0)*100.0))
- }
- return append(table, row)
-}
-
-func metricOf(unit string) string {
- switch unit {
- case "ns/op":
- return "time/op"
- case "B/op":
- return "alloc/op"
- case "MB/s":
- return "speed"
- default:
- return unit
- }
-}
diff --git a/cmd/benchstat/table.go b/cmd/benchstat/table.go
new file mode 100644
index 0000000..51be88c
--- /dev/null
+++ b/cmd/benchstat/table.go
@@ -0,0 +1,149 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "fmt"
+
+ "golang.org/x/perf/internal/stats"
+)
+
+// A Table is a table for display in the benchstat output.
+type Table struct {
+ Metric string
+ Configs []string
+ Rows []*Row
+}
+
+// A Row is a table row for display in the benchstat output.
+type Row struct {
+ Benchmark string // benchmark name
+ Scaler Scaler // formatter for stats means
+ Metrics []*Metrics // columns of statistics (nil slice entry means no data)
+ Delta string // formatted percent change
+ Note string // additional information
+ Same bool // likely no change
+}
+
+// Tables returns tables comparing the benchmarks in the collection.
+func (c *Collection) Tables(deltaTest DeltaTest) []*Table {
+ // Update statistics.
+ for _, m := range c.Metrics {
+ m.computeStats()
+ }
+
+ var tables []*Table
+ key := Key{}
+ for _, key.Unit = range c.Units {
+ table := new(Table)
+ table.Configs = c.Configs
+ table.Metric = metricOf(key.Unit)
+ for _, key.Benchmark = range c.Benchmarks {
+ row := &Row{Benchmark: key.Benchmark}
+
+ for _, key.Config = range c.Configs {
+ m := c.Metrics[key]
+ row.Metrics = append(row.Metrics, m)
+ if m == nil {
+ continue
+ }
+ if row.Scaler == nil {
+ row.Scaler = NewScaler(m.Mean, m.Unit)
+ }
+ }
+
+ // If there are only two configs being compared, add stats.
+ // If one is missing, omit line entirely.
+ // TODO: Control this better.
+ if len(c.Configs) == 2 {
+ k0 := key
+ k0.Config = c.Configs[0]
+ k1 := key
+ k1.Config = c.Configs[1]
+ old := c.Metrics[k0]
+ new := c.Metrics[k1]
+ if old == nil || new == nil {
+ continue
+ }
+ pval, testerr := deltaTest(old, new)
+ row.Delta = "~"
+ if testerr == stats.ErrZeroVariance {
+ row.Note = "(zero variance)"
+ } else if testerr == stats.ErrSampleSize {
+ row.Note = "(too few samples)"
+ } else if testerr == stats.ErrSamplesEqual {
+ row.Note = "(all equal)"
+ } else if testerr != nil {
+ row.Note = fmt.Sprintf("(%s)", testerr)
+ } else if pval < *flagAlpha {
+ row.Delta = fmt.Sprintf("%+.2f%%", ((new.Mean/old.Mean)-1.0)*100.0)
+ }
+ if row.Note == "" && pval != -1 {
+ row.Note = fmt.Sprintf("(p=%0.3f n=%d+%d)", pval, len(old.RValues), len(new.RValues))
+ }
+ }
+
+ table.Rows = append(table.Rows, row)
+ }
+
+ if len(table.Rows) > 0 {
+ if *flagGeomean {
+ addGeomean(c, table, key.Unit, len(c.Configs) == 2)
+ }
+ tables = append(tables, table)
+ }
+ }
+
+ return tables
+}
+
+// metricOf returns the name of the metric with the given unit.
+func metricOf(unit string) string {
+ switch unit {
+ case "ns/op":
+ return "time/op"
+ case "B/op":
+ return "alloc/op"
+ case "MB/s":
+ return "speed"
+ default:
+ return unit
+ }
+}
+
+// addGeomean adds a "geomean" row to the table,
+// showing the geometric mean of all the benchmarks.
+func addGeomean(c *Collection, t *Table, unit string, delta bool) {
+ row := &Row{Benchmark: "[Geo mean]"}
+ key := Key{Unit: unit}
+ geomeans := []float64{}
+ for _, key.Config = range c.Configs {
+ var means []float64
+ for _, key.Benchmark = range c.Benchmarks {
+ m := c.Metrics[key]
+ if m != nil {
+ means = append(means, m.Mean)
+ }
+ }
+ if len(means) == 0 {
+ row.Metrics = append(row.Metrics, nil)
+ delta = false
+ } else {
+ geomean := stats.GeoMean(means)
+ geomeans = append(geomeans, geomean)
+ if row.Scaler == nil {
+ row.Scaler = NewScaler(geomean, unit)
+ }
+ row.Metrics = append(row.Metrics, &Metrics{
+ Unit: unit,
+ Mean: geomean,
+ })
+ }
+ }
+ if delta {
+ row.Delta = fmt.Sprintf("%+.2f%%", ((geomeans[1]/geomeans[0])-1.0)*100.0)
+ }
+ t.Rows = append(t.Rows, row)
+}
diff --git a/cmd/benchstat/text.go b/cmd/benchstat/text.go
new file mode 100644
index 0000000..2654344
--- /dev/null
+++ b/cmd/benchstat/text.go
@@ -0,0 +1,126 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "unicode/utf8"
+)
+
+// FormatText appends a fixed-width text formatting of the tables to buf.
+func FormatText(buf *bytes.Buffer, tables []*Table) {
+ var textTables [][]*textRow
+ for _, t := range tables {
+ textTables = append(textTables, toText(t))
+ }
+
+ var max []int
+ for _, table := range textTables {
+ for _, row := range table {
+ for len(max) < len(row.cols) {
+ max = append(max, 0)
+ }
+ for i, s := range row.cols {
+ n := utf8.RuneCountInString(s)
+ if max[i] < n {
+ max[i] = n
+ }
+ }
+ }
+ }
+
+ for i, table := range textTables {
+ if i > 0 {
+ fmt.Fprintf(buf, "\n")
+ }
+
+ // headings
+ row := table[0]
+ for i, s := range row.cols {
+ switch i {
+ case 0:
+ fmt.Fprintf(buf, "%-*s", max[i], s)
+ default:
+ fmt.Fprintf(buf, " %-*s", max[i], s)
+ case len(row.cols) - 1:
+ fmt.Fprintf(buf, " %s\n", s)
+ }
+ }
+
+ // data
+ for _, row := range table[1:] {
+ for i, s := range row.cols {
+ switch i {
+ case 0:
+ fmt.Fprintf(buf, "%-*s", max[i], s)
+ default:
+ if i == len(row.cols)-1 && len(s) > 0 && s[0] == '(' {
+ // Left-align p value.
+ fmt.Fprintf(buf, " %s", s)
+ break
+ }
+ fmt.Fprintf(buf, " %*s", max[i], s)
+ }
+ }
+ fmt.Fprintf(buf, "\n")
+ }
+ }
+}
+
+// A textRow is a row of printed text columns.
+type textRow struct {
+ cols []string
+}
+
+func newTextRow(cols ...string) *textRow {
+ return &textRow{cols: cols}
+}
+
+func (r *textRow) add(col string) {
+ r.cols = append(r.cols, col)
+}
+
+func (r *textRow) trim() {
+ for len(r.cols) > 0 && r.cols[len(r.cols)-1] == "" {
+ r.cols = r.cols[:len(r.cols)-1]
+ }
+}
+
+// toText converts the Table to a textual grid of cells,
+// which can then be printed in fixed-width output.
+func toText(t *Table) []*textRow {
+ var textRows []*textRow
+ switch len(t.Configs) {
+ case 1:
+ textRows = append(textRows, newTextRow("name", t.Metric))
+ case 2:
+ textRows = append(textRows, newTextRow("name", "old "+t.Metric, "new "+t.Metric, "delta"))
+ default:
+ row := newTextRow("name \\ " + t.Metric)
+ row.cols = append(row.cols, t.Configs...)
+ textRows = append(textRows, row)
+ }
+
+ for _, row := range t.Rows {
+ text := newTextRow(row.Benchmark)
+ for _, m := range row.Metrics {
+ text.cols = append(text.cols, m.Format(row.Scaler))
+ }
+ if len(t.Configs) == 2 {
+ delta := row.Delta
+ if delta == "~" {
+ delta = "~ "
+ }
+ text.cols = append(text.cols, delta)
+ text.cols = append(text.cols, row.Note)
+ }
+ textRows = append(textRows, text)
+ }
+ for _, r := range textRows {
+ r.trim()
+ }
+ return textRows
+}
