Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 1 | // Copyright 2017 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
Russ Cox | c1e5ad7 | 2017-01-27 16:34:00 -0500 | [diff] [blame] | 5 | package benchstat |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 6 | |
| 7 | import ( |
| 8 | "fmt" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | |
| 12 | "golang.org/x/perf/internal/stats" |
Russ Cox | 66643a1 | 2017-01-27 16:38:35 -0500 | [diff] [blame] | 13 | "golang.org/x/perf/storage/benchfmt" |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 14 | ) |
| 15 | |
| 16 | // A Collection is a collection of benchmark results. |
| 17 | type Collection struct { |
| 18 | // Configs, Benchmarks, and Units give the set of configs, |
| 19 | // benchmarks, and units from the keys in Stats in an order |
| 20 | // meant to match the order the benchmarks were read in. |
| 21 | Configs, Benchmarks, Units []string |
| 22 | |
| 23 | // Metrics holds the accumulated metrics for each key. |
| 24 | Metrics map[Key]*Metrics |
Russ Cox | c1e5ad7 | 2017-01-27 16:34:00 -0500 | [diff] [blame] | 25 | |
| 26 | // DeltaTest is the test to use to decide if a change is significant. |
| 27 | // If nil, it defaults to UTest. |
| 28 | DeltaTest DeltaTest |
| 29 | |
| 30 | // Alpha is the p-value cutoff to report a change as significant. |
| 31 | // If zero, it defaults to 0.05. |
| 32 | Alpha float64 |
| 33 | |
| 34 | // AddGeoMean specifies whether to add a line to the table |
| 35 | // showing the geometric mean of all the benchmark results. |
| 36 | AddGeoMean bool |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 37 | } |
| 38 | |
| 39 | // A Key identifies one metric (e.g., "ns/op", "B/op") from one |
| 40 | // benchmark (function name sans "Benchmark" prefix) in one |
| 41 | // configuration (input file name). |
| 42 | type Key struct { |
| 43 | Config, Benchmark, Unit string |
| 44 | } |
| 45 | |
| 46 | // A Metrics holds the measurements of a single metric |
| 47 | // (for example, ns/op or MB/s) |
| 48 | // for all runs of a particular benchmark. |
| 49 | type Metrics struct { |
| 50 | Unit string // unit being measured |
| 51 | Values []float64 // measured values |
| 52 | RValues []float64 // Values with outliers removed |
| 53 | Min float64 // min of RValues |
| 54 | Mean float64 // mean of RValues |
| 55 | Max float64 // max of RValues |
| 56 | } |
| 57 | |
| 58 | // FormatMean formats m.Mean using scaler. |
| 59 | func (m *Metrics) FormatMean(scaler Scaler) string { |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 60 | var s string |
| 61 | if scaler != nil { |
| 62 | s = scaler(m.Mean) |
| 63 | } else { |
| 64 | s = fmt.Sprint(m.Mean) |
| 65 | } |
| 66 | return s |
| 67 | } |
| 68 | |
| 69 | // FormatDiff computes and formats the percent variation of max and min compared to mean. |
| 70 | // If b.Mean or b.Max is zero, FormatDiff returns an empty string. |
| 71 | func (m *Metrics) FormatDiff() string { |
Russ Cox | 1b4493f | 2017-01-27 14:35:35 -0500 | [diff] [blame] | 72 | if m.Mean == 0 || m.Max == 0 { |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 73 | return "" |
| 74 | } |
| 75 | diff := 1 - m.Min/m.Mean |
| 76 | if d := m.Max/m.Mean - 1; d > diff { |
| 77 | diff = d |
| 78 | } |
| 79 | return fmt.Sprintf("%.0f%%", diff*100.0) |
| 80 | } |
| 81 | |
| 82 | // Format returns a textual formatting of "Mean ±Diff" using scaler. |
| 83 | func (m *Metrics) Format(scaler Scaler) string { |
Russ Cox | 1b4493f | 2017-01-27 14:35:35 -0500 | [diff] [blame] | 84 | if m.Unit == "" { |
Russ Cox | a7a7b5b | 2017-01-27 12:19:00 -0500 | [diff] [blame] | 85 | return "" |
| 86 | } |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 87 | mean := m.FormatMean(scaler) |
| 88 | diff := m.FormatDiff() |
| 89 | if diff == "" { |
| 90 | return mean + " " |
| 91 | } |
| 92 | return fmt.Sprintf("%s ±%3s", mean, diff) |
| 93 | } |
| 94 | |
| 95 | // computeStats updates the derived statistics in m from the raw |
| 96 | // samples in m.Values. |
| 97 | func (m *Metrics) computeStats() { |
| 98 | // Discard outliers. |
| 99 | values := stats.Sample{Xs: m.Values} |
| 100 | q1, q3 := values.Percentile(0.25), values.Percentile(0.75) |
| 101 | lo, hi := q1-1.5*(q3-q1), q3+1.5*(q3-q1) |
| 102 | for _, value := range m.Values { |
| 103 | if lo <= value && value <= hi { |
| 104 | m.RValues = append(m.RValues, value) |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | // Compute statistics of remaining data. |
| 109 | m.Min, m.Max = stats.Bounds(m.RValues) |
| 110 | m.Mean = stats.Mean(m.RValues) |
| 111 | } |
| 112 | |
| 113 | // addMetrics returns the metrics with the given key from c, |
| 114 | // creating a new one if needed. |
| 115 | func (c *Collection) addMetrics(key Key) *Metrics { |
| 116 | if c.Metrics == nil { |
| 117 | c.Metrics = make(map[Key]*Metrics) |
| 118 | } |
| 119 | if stat, ok := c.Metrics[key]; ok { |
| 120 | return stat |
| 121 | } |
| 122 | |
| 123 | addString := func(strings *[]string, add string) { |
| 124 | for _, s := range *strings { |
| 125 | if s == add { |
| 126 | return |
| 127 | } |
| 128 | } |
| 129 | *strings = append(*strings, add) |
| 130 | } |
| 131 | addString(&c.Configs, key.Config) |
| 132 | addString(&c.Benchmarks, key.Benchmark) |
| 133 | addString(&c.Units, key.Unit) |
| 134 | m := &Metrics{Unit: key.Unit} |
| 135 | c.Metrics[key] = m |
| 136 | return m |
| 137 | } |
| 138 | |
Russ Cox | 66643a1 | 2017-01-27 16:38:35 -0500 | [diff] [blame] | 139 | // AddFile adds the benchmark results in the formatted data |
| 140 | // to the named configuration. |
Russ Cox | ae16c2a | 2017-02-01 09:51:48 -0500 | [diff] [blame] | 141 | func (c *Collection) AddConfig(config string, data []byte) { |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 142 | c.Configs = append(c.Configs, config) |
| 143 | key := Key{Config: config} |
Russ Cox | 66643a1 | 2017-01-27 16:38:35 -0500 | [diff] [blame] | 144 | c.addText(key, string(data)) |
| 145 | } |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 146 | |
Russ Cox | 66643a1 | 2017-01-27 16:38:35 -0500 | [diff] [blame] | 147 | // AddResults adds the benchmark results to the named configuration. |
| 148 | func (c *Collection) AddResults(config string, results []*benchfmt.Result) { |
| 149 | c.Configs = append(c.Configs, config) |
| 150 | key := Key{Config: config} |
| 151 | for _, r := range results { |
| 152 | c.addText(key, r.Content) |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | func (c *Collection) addText(key Key, data string) { |
Russ Cox | b4c600e | 2017-01-27 12:11:31 -0500 | [diff] [blame] | 157 | for _, line := range strings.Split(string(data), "\n") { |
| 158 | f := strings.Fields(line) |
| 159 | if len(f) < 4 { |
| 160 | continue |
| 161 | } |
| 162 | name := f[0] |
| 163 | if !strings.HasPrefix(name, "Benchmark") { |
| 164 | continue |
| 165 | } |
| 166 | name = strings.TrimPrefix(name, "Benchmark") |
| 167 | n, _ := strconv.Atoi(f[1]) |
| 168 | if n == 0 { |
| 169 | continue |
| 170 | } |
| 171 | |
| 172 | key.Benchmark = name |
| 173 | for i := 2; i+2 <= len(f); i += 2 { |
| 174 | val, err := strconv.ParseFloat(f[i], 64) |
| 175 | if err != nil { |
| 176 | continue |
| 177 | } |
| 178 | key.Unit = f[i+1] |
| 179 | m := c.addMetrics(key) |
| 180 | m.Values = append(m.Values, val) |
| 181 | } |
| 182 | } |
| 183 | } |