blob: 756e2b067d6a5a5e8ac0bb9756c99d194b20b155 [file] [log] [blame]
Russ Coxb4c600e2017-01-27 12:11:31 -05001// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
Russ Coxc1e5ad72017-01-27 16:34:00 -05005package benchstat
Russ Coxb4c600e2017-01-27 12:11:31 -05006
7import (
8 "fmt"
9 "strconv"
10 "strings"
11
12 "golang.org/x/perf/internal/stats"
Russ Cox66643a12017-01-27 16:38:35 -050013 "golang.org/x/perf/storage/benchfmt"
Russ Coxb4c600e2017-01-27 12:11:31 -050014)
15
16// A Collection is a collection of benchmark results.
17type Collection struct {
18 // Configs, Benchmarks, and Units give the set of configs,
19 // benchmarks, and units from the keys in Stats in an order
20 // meant to match the order the benchmarks were read in.
21 Configs, Benchmarks, Units []string
22
23 // Metrics holds the accumulated metrics for each key.
24 Metrics map[Key]*Metrics
Russ Coxc1e5ad72017-01-27 16:34:00 -050025
26 // DeltaTest is the test to use to decide if a change is significant.
27 // If nil, it defaults to UTest.
28 DeltaTest DeltaTest
29
30 // Alpha is the p-value cutoff to report a change as significant.
31 // If zero, it defaults to 0.05.
32 Alpha float64
33
34 // AddGeoMean specifies whether to add a line to the table
35 // showing the geometric mean of all the benchmark results.
36 AddGeoMean bool
Russ Coxb4c600e2017-01-27 12:11:31 -050037}
38
39// A Key identifies one metric (e.g., "ns/op", "B/op") from one
40// benchmark (function name sans "Benchmark" prefix) in one
41// configuration (input file name).
42type Key struct {
43 Config, Benchmark, Unit string
44}
45
46// A Metrics holds the measurements of a single metric
47// (for example, ns/op or MB/s)
48// for all runs of a particular benchmark.
49type Metrics struct {
50 Unit string // unit being measured
51 Values []float64 // measured values
52 RValues []float64 // Values with outliers removed
53 Min float64 // min of RValues
54 Mean float64 // mean of RValues
55 Max float64 // max of RValues
56}
57
58// FormatMean formats m.Mean using scaler.
59func (m *Metrics) FormatMean(scaler Scaler) string {
Russ Coxb4c600e2017-01-27 12:11:31 -050060 var s string
61 if scaler != nil {
62 s = scaler(m.Mean)
63 } else {
64 s = fmt.Sprint(m.Mean)
65 }
66 return s
67}
68
69// FormatDiff computes and formats the percent variation of max and min compared to mean.
70// If b.Mean or b.Max is zero, FormatDiff returns an empty string.
71func (m *Metrics) FormatDiff() string {
Russ Cox1b4493f2017-01-27 14:35:35 -050072 if m.Mean == 0 || m.Max == 0 {
Russ Coxb4c600e2017-01-27 12:11:31 -050073 return ""
74 }
75 diff := 1 - m.Min/m.Mean
76 if d := m.Max/m.Mean - 1; d > diff {
77 diff = d
78 }
79 return fmt.Sprintf("%.0f%%", diff*100.0)
80}
81
82// Format returns a textual formatting of "Mean ±Diff" using scaler.
83func (m *Metrics) Format(scaler Scaler) string {
Russ Cox1b4493f2017-01-27 14:35:35 -050084 if m.Unit == "" {
Russ Coxa7a7b5b2017-01-27 12:19:00 -050085 return ""
86 }
Russ Coxb4c600e2017-01-27 12:11:31 -050087 mean := m.FormatMean(scaler)
88 diff := m.FormatDiff()
89 if diff == "" {
90 return mean + " "
91 }
92 return fmt.Sprintf("%s ±%3s", mean, diff)
93}
94
95// computeStats updates the derived statistics in m from the raw
96// samples in m.Values.
97func (m *Metrics) computeStats() {
98 // Discard outliers.
99 values := stats.Sample{Xs: m.Values}
100 q1, q3 := values.Percentile(0.25), values.Percentile(0.75)
101 lo, hi := q1-1.5*(q3-q1), q3+1.5*(q3-q1)
102 for _, value := range m.Values {
103 if lo <= value && value <= hi {
104 m.RValues = append(m.RValues, value)
105 }
106 }
107
108 // Compute statistics of remaining data.
109 m.Min, m.Max = stats.Bounds(m.RValues)
110 m.Mean = stats.Mean(m.RValues)
111}
112
113// addMetrics returns the metrics with the given key from c,
114// creating a new one if needed.
115func (c *Collection) addMetrics(key Key) *Metrics {
116 if c.Metrics == nil {
117 c.Metrics = make(map[Key]*Metrics)
118 }
119 if stat, ok := c.Metrics[key]; ok {
120 return stat
121 }
122
123 addString := func(strings *[]string, add string) {
124 for _, s := range *strings {
125 if s == add {
126 return
127 }
128 }
129 *strings = append(*strings, add)
130 }
131 addString(&c.Configs, key.Config)
132 addString(&c.Benchmarks, key.Benchmark)
133 addString(&c.Units, key.Unit)
134 m := &Metrics{Unit: key.Unit}
135 c.Metrics[key] = m
136 return m
137}
138
Russ Cox66643a12017-01-27 16:38:35 -0500139// AddFile adds the benchmark results in the formatted data
140// to the named configuration.
Russ Coxae16c2a2017-02-01 09:51:48 -0500141func (c *Collection) AddConfig(config string, data []byte) {
Russ Coxb4c600e2017-01-27 12:11:31 -0500142 c.Configs = append(c.Configs, config)
143 key := Key{Config: config}
Russ Cox66643a12017-01-27 16:38:35 -0500144 c.addText(key, string(data))
145}
Russ Coxb4c600e2017-01-27 12:11:31 -0500146
Russ Cox66643a12017-01-27 16:38:35 -0500147// AddResults adds the benchmark results to the named configuration.
148func (c *Collection) AddResults(config string, results []*benchfmt.Result) {
149 c.Configs = append(c.Configs, config)
150 key := Key{Config: config}
151 for _, r := range results {
152 c.addText(key, r.Content)
153 }
154}
155
156func (c *Collection) addText(key Key, data string) {
Russ Coxb4c600e2017-01-27 12:11:31 -0500157 for _, line := range strings.Split(string(data), "\n") {
158 f := strings.Fields(line)
159 if len(f) < 4 {
160 continue
161 }
162 name := f[0]
163 if !strings.HasPrefix(name, "Benchmark") {
164 continue
165 }
166 name = strings.TrimPrefix(name, "Benchmark")
167 n, _ := strconv.Atoi(f[1])
168 if n == 0 {
169 continue
170 }
171
172 key.Benchmark = name
173 for i := 2; i+2 <= len(f); i += 2 {
174 val, err := strconv.ParseFloat(f[i], 64)
175 if err != nil {
176 continue
177 }
178 key.Unit = f[i+1]
179 m := c.addMetrics(key)
180 m.Values = append(m.Values, val)
181 }
182 }
183}