cmd/bent: add flag to disable build time reporting
We do not want these benchmarks in cmd/bench because we only run builds
once and thus do not get statistically significant results anyway.
For golang/go#57770.
Change-Id: I75987416458f826ee0a3e71c69f36fd48206eb77
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/463196
Run-TryBot: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Pratt <mpratt@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>
diff --git a/cmd/bench/bent.go b/cmd/bench/bent.go
index 2f87656..25152fe 100644
--- a/cmd/bench/bent.go
+++ b/cmd/bench/bent.go
@@ -112,6 +112,7 @@
"-N", "10",
"-C", confFile,
"-B", filepath.Join(dir, "benchmarks-50.toml"),
+ "-report-build-time=false", // We only run builds once, which won't yield statistically significant results.
"-v",
)
cmd.Dir = dir
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index fb411c1..72e8e97 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -89,6 +89,7 @@
var explicitAll counterFlag // Include "-a" on "go test -c" test build ; repeating flag causes multiple rebuilds, useful for build benchmarking.
var shuffle = 2 // Dimensionality of (build) shuffling; 0 = none, 1 = per-benchmark, configuration ordering, 2 = bench, config pairs, 3 = across repetitions.
var haveRsync = true
+var reportBuildTime = true
//go:embed scripts/*
var scripts embed.FS
@@ -160,6 +161,8 @@
flag.BoolVar(&wikiTable, "W", wikiTable, "print benchmark info for a wiki table")
+ flag.BoolVar(&reportBuildTime, "report-build-time", reportBuildTime, "report build real/CPU time as benchmark results")
+
flag.Var(&verbose, "v", "print commands and other information (more -v = print more details)")
flag.Usage = func() {
diff --git a/cmd/bent/configuration.go b/cmd/bent/configuration.go
index bd192be..67873aa 100644
--- a/cmd/bent/configuration.go
+++ b/cmd/bent/configuration.go
@@ -35,7 +35,6 @@
RunEnv []string // Extra environment variables passed to the test binary
RunWrapper []string // (Outermost) Command and args to precede whatever the operation is; may fail in the sandbox.
Disabled bool // True if this configuration is temporarily disabled
- buildStats []BenchStat
benchWriter *os.File
rootCopy string // The contents of GOROOT are copied here to allow benchmarking of just the test compilation.
}
@@ -236,49 +235,50 @@
bench.Disabled = true // if it won't compile, it won't run, either.
return s + "(" + bench.Name + ")\n"
}
- soutput := string(output)
- bs := BenchStat{
- Name: bench.Name,
- RealTime: realTime,
- UserTime: cmd.ProcessState.UserTime(),
- SysTime: cmd.ProcessState.SystemTime(),
- }
- config.buildStats = append(config.buildStats, bs)
- // Report and record build stats to testbin
+ if reportBuildTime {
+ // Report and record build stats to testbin
+ bs := BenchStat{
+ Name: bench.Name,
+ RealTime: realTime,
+ UserTime: cmd.ProcessState.UserTime(),
+ SysTime: cmd.ProcessState.SystemTime(),
+ }
- buf := new(bytes.Buffer)
- var goarchVal string
- if configGoArch != runtime.GOARCH && configGoArch != "" {
- goarchVal = fmt.Sprintf("%s-%s", runtime.GOARCH, configGoArch)
- } else {
- goarchVal = runtime.GOARCH
- }
- var s string
- s += fmt.Sprintf("goarch: %s\n", goarchVal)
- s += fmt.Sprintf("toolchain: %s\n", config.Name)
- if verbose > 0 {
- fmt.Print(s)
- }
- buf.WriteString(s)
- s = fmt.Sprintf("Benchmark%s 1 %d build-real-ns/op %d build-user-ns/op %d build-sys-ns/op\n",
+ buf := new(bytes.Buffer)
+ var goarchVal string
+ if configGoArch != runtime.GOARCH && configGoArch != "" {
+ goarchVal = fmt.Sprintf("%s-%s", runtime.GOARCH, configGoArch)
+ } else {
+ goarchVal = runtime.GOARCH
+ }
+ var s string
+ s += fmt.Sprintf("goarch: %s\n", goarchVal)
+ s += fmt.Sprintf("toolchain: %s\n", config.Name)
+ if verbose > 0 {
+ fmt.Print(s)
+ }
+ buf.WriteString(s)
+ s = fmt.Sprintf("Benchmark%s 1 %d build-real-ns/op %d build-user-ns/op %d build-sys-ns/op\n",
strings.Title(bench.Name), bs.RealTime.Nanoseconds(), bs.UserTime.Nanoseconds(), bs.SysTime.Nanoseconds())
- if verbose > 0 {
- fmt.Print(s)
+ if verbose > 0 {
+ fmt.Print(s)
+ }
+ buf.WriteString(s)
+ f, err := os.OpenFile(config.buildBenchName(), os.O_WRONLY|os.O_APPEND, os.ModePerm)
+ if err != nil {
+ fmt.Printf("There was an error opening %s for append, error %v\n", config.buildBenchName(), err)
+ cleanup(gopath)
+ os.Exit(2)
+ }
+ f.Write(buf.Bytes())
+ f.Sync()
+ f.Close()
}
- buf.WriteString(s)
- f, err := os.OpenFile(config.buildBenchName(), os.O_WRONLY|os.O_APPEND, os.ModePerm)
- if err != nil {
- fmt.Printf("There was an error opening %s for append, error %v\n", config.buildBenchName(), err)
- cleanup(gopath)
- os.Exit(2)
- }
- f.Write(buf.Bytes())
- f.Sync()
- f.Close()
// Trim /usr/bin/time info from soutput, it's ugly
if verbose > 0 {
+ soutput := string(output)
i := strings.LastIndex(soutput, "real")
if i >= 0 {
soutput = soutput[:i]