all: report in standard benchmark format
This switches all benchmarks to report in the standard Go benchmark
format [1], making it possible to run and analyze these benchmarks
using tools built for standard Go benchmarks.
As part of this, we lift the repetition loop to the top level to make
it possible to report the results after each run.
[1] https://github.com/golang/proposal/blob/master/design/14313-benchmark-format.md
Change-Id: I542e0f4f6fcfa7783ee37bcdc7edf644201b0b60
Reviewed-on: https://go-review.googlesource.com/33581
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>
Reviewed-by: Russ Cox <rsc@golang.org>
diff --git a/build/build.go b/build/build.go
index 302bdd8..0841498 100644
--- a/build/build.go
+++ b/build/build.go
@@ -15,21 +15,14 @@
)
func main() {
- driver.Main(benchmark)
+ driver.Main("Build", benchmark)
}
func benchmark() driver.Result {
if os.Getenv("GOMAXPROCS") == "" {
os.Setenv("GOMAXPROCS", "1")
}
- res := driver.MakeResult()
- for i := 0; i < driver.BenchNum; i++ {
- res1 := benchmarkOnce()
- if res.RunTime == 0 || res.RunTime > res1.RunTime {
- res = res1
- }
- log.Printf("Run %v: %+v\n", i, res)
- }
+ res := benchmarkOnce()
perf1, perf2 := driver.RunUnderProfiler("go", "build", "-o", "goperf", "-a", "-p", os.Getenv("GOMAXPROCS"), "cmd/go")
if perf1 != "" {
res.Files["processes"] = perf1
diff --git a/driver/driver.go b/driver/driver.go
index d407724..d5060a7 100644
--- a/driver/driver.go
+++ b/driver/driver.go
@@ -23,7 +23,6 @@
"runtime"
"runtime/pprof"
"sort"
- "strings"
"sync"
"sync/atomic"
"time"
@@ -31,14 +30,13 @@
var (
flake = flag.Int("flake", 0, "test flakiness of a benchmark")
- benchNum = flag.Int("benchnum", 5, "number of benchmark runs")
+ benchNum = flag.Int("benchnum", 1, "number of benchmark runs")
benchMem = flag.Int("benchmem", 64, "approx RSS value to aim at in benchmarks, in MB")
benchTime = flag.Duration("benchtime", 5*time.Second, "run enough iterations of each benchmark to take the specified time")
affinity = flag.Int("affinity", 0, "process affinity (passed to an OS-specific function like sched_setaffinity/SetProcessAffinityMask)")
tmpDir = flag.String("tmpdir", os.TempDir(), "dir for temporary files")
genSvg = flag.Bool("svg", false, "generate svg profiles")
- BenchNum int
BenchMem int
BenchTime time.Duration
WorkDir string
@@ -50,10 +48,9 @@
}
)
-func Main(f func() Result) {
+func Main(name string, f func() Result) {
flag.Parse()
// Copy to public variables, so that benchmarks can access the values.
- BenchNum = *benchNum
BenchMem = *benchMem
BenchTime = *benchTime
WorkDir = *tmpDir
@@ -72,24 +69,9 @@
stopTrace := startTrace()
defer stopTrace()
- res := f()
-
- var metrics []string
- for k := range res.Metrics {
- metrics = append(metrics, k)
- }
- sort.Strings(metrics)
- for _, m := range metrics {
- fmt.Printf("GOPERF-METRIC:%v=%v\n", m, res.Metrics[m])
- }
-
- var files []string
- for k := range res.Files {
- files = append(files, k)
- }
- sort.Strings(files)
- for _, f := range files {
- fmt.Printf("GOPERF-FILE:%v=%v\n", f, res.Files[f])
+ for i := 0; i < *benchNum; i++ {
+ res := f()
+ report(name, res)
}
}
@@ -141,24 +123,31 @@
return Result{Metrics: make(map[string]uint64), Files: make(map[string]string)}
}
-// Benchmark runs f several times, collects stats, chooses the best run
+func report(name string, res Result) {
+ for name, path := range res.Files {
+ fmt.Printf("# %s=%s\n", name, path)
+ }
+
+ fmt.Printf("Benchmark%s-%d %8d\t%10d ns/op", name, runtime.GOMAXPROCS(-1), res.N, res.RunTime)
+ var metrics []string
+ for metric := range res.Metrics {
+ if metric == "time" {
+ // Already reported from res.RunTime.
+ continue
+ }
+ metrics = append(metrics, metric)
+ }
+ sort.Strings(metrics)
+ for _, metric := range metrics {
+ fmt.Printf("\t%10d %s", res.Metrics[metric], metric)
+ }
+ fmt.Printf("\n")
+}
+
+// Benchmark runs f several times, collects stats,
// and creates cpu/mem profiles.
func Benchmark(f func(uint64)) Result {
- res := MakeResult()
- for i := 0; i < *benchNum; i++ {
- res1 := runBenchmark(f)
- if res.N == 0 || res.RunTime > res1.RunTime {
- res = res1
- }
- // Always take RSS and sys memory metrics from last iteration.
- // They only grow, and seem to converge to some eigen value.
- // Variations are smaller if we do this.
- for k, v := range res1.Metrics {
- if k == "rss" || strings.HasPrefix(k, "sys-") {
- res.Metrics[k] = v
- }
- }
- }
+ res := runBenchmark(f)
cpuprof := processProfile(os.Args[0], res.Files["cpuprof"])
delete(res.Files, "cpuprof")
@@ -212,7 +201,6 @@
log.Printf("Benchmarking %v iterations\n", res.N)
res = runBenchmarkOnce(f, res.N)
}
- log.Printf("Result: %+v\n", res)
return res
}
diff --git a/garbage/garbage.go b/garbage/garbage.go
index f1ac2d6..8378a55 100644
--- a/garbage/garbage.go
+++ b/garbage/garbage.go
@@ -22,7 +22,7 @@
)
func main() {
- driver.Main(benchmark)
+ driver.Main("Garbage", benchmark)
}
// func init() {
diff --git a/http/http.go b/http/http.go
index 97d5f8f..1549281 100644
--- a/http/http.go
+++ b/http/http.go
@@ -19,7 +19,7 @@
)
func main() {
- driver.Main(benchmark)
+ driver.Main("HTTP", benchmark)
}
func benchmark() driver.Result {
diff --git a/json/json.go b/json/json.go
index ffc5a11..58269bd 100644
--- a/json/json.go
+++ b/json/json.go
@@ -18,7 +18,7 @@
)
func main() {
- driver.Main(benchmark)
+ driver.Main("JSON", benchmark)
}
func benchmark() driver.Result {
