cmd/bent: refactor benchmark running pull the runner out into a single function. fix a bug in the PGO feature Added toml benchmark. Change-Id: I3be36baddb5a7ca258a1a6872351af5146a1d641 Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/565115 LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com> Reviewed-by: Michael Knyszek <mknyszek@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go index b2be732..704b0cf 100644 --- a/cmd/bent/bent.go +++ b/cmd/bent/bent.go
@@ -344,6 +344,9 @@ // TODO should it use concatenation of os env and configuration env? trial.RunWrapper[0] = os.ExpandEnv(trial.RunWrapper[0]) } + // TODO would anyone ever make these depend on BENT_I etc? + trial.PgoGen = os.ExpandEnv(trial.PgoGen) + trial.PgoUse = os.ExpandEnv(trial.PgoUse) } for b, v := range configurations { if v { @@ -966,128 +969,20 @@ // TODO randomize the benchmarks and configurations, like for builds. for i := 0; i < N; i++ { // For each configuration, run all the benchmarks. - for j, config := range todo.Configurations { - if config.Disabled { + for j := range todo.Configurations { + c := &todo.Configurations[j] + if c.Disabled { continue } - for _, b := range todo.Benchmarks { + for k := range todo.Benchmarks { + b := &todo.Benchmarks[k] if b.Disabled { continue } - root := config.Root + s, rc := benchOne(c, b, i, moreArgs) - wrapperPrefix := "/" - if b.NotSandboxed { - wrapperPrefix = dirs.wd + "/" - } - wrapperFor := func(s []string) string { - x := "" - if len(s) > 0 { - // If not an explicit path, then make it an explicit path - x = s[0] - if x[0] != '/' { - x = wrapperPrefix + x - } - } - return x - } - - testBinaryName := config.benchName(&b) - - runEnv := []string{} - runEnv = append(runEnv, "BENT_CONFIG="+config.Name) - runEnv = append(runEnv, "BENT_BENCH="+b.Name) - runEnv = append(runEnv, "BENT_I="+strconv.FormatInt(int64(i), 10)) - runEnv = append(runEnv, "BENT_BINARY="+testBinaryName) - - if config.PgoGen != "" { - // We want to generate pprof file for using pgo - config.RunWrapper = append(config.RunWrapper, "cpuprofile") - } - - configWrapper := wrapperFor(config.RunWrapper) - benchWrapper := wrapperFor(b.RunWrapper) - - var s string - var rc int - - var wrappersAndBin []string - - if configWrapper != "" { - wrappersAndBin = append(wrappersAndBin, configWrapper) - wrappersAndBin = append(wrappersAndBin, config.RunWrapper[1:]...) - } - if benchWrapper != "" { - wrappersAndBin = append(wrappersAndBin, benchWrapper) - wrappersAndBin = append(wrappersAndBin, b.RunWrapper[1:]...) - } - - if b.NotSandboxed { - bin := path.Join(dirs.wd, dirs.testBinDir, testBinaryName) - wrappersAndBin = append(wrappersAndBin, bin) - - cmd := exec.Command(wrappersAndBin[0], wrappersAndBin[1:]...) - cmd.Args = append(cmd.Args, "-test.run="+b.Tests) - cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks) - - cmd.Dir = b.RunDir - cmd.Env = DefaultEnv() - if root != "" { - cmd.Env = replaceEnv(cmd.Env, "GOROOT", root) - } - cmd.Env = append(cmd.Env, "BENT_PROFILES="+path.Join(dirs.wd, config.thingBenchName("profiles"))) - if config.PgoGen != "" { - // We want to generate pprof file for using pgo - cmd.Env = append(cmd.Env, "BENT_PGO="+path.Join(dirs.wd, config.PgoGen)) - } - - cmd.Env = append(cmd.Env, runEnv...) - cmd.Env = append(cmd.Env, sliceExpandEnv(config.RunEnv, cmd.Env)...) - - cmd.Args = append(cmd.Args, config.RunFlags...) - cmd.Args = append(cmd.Args, moreArgs...) - cmd.Args = sliceExpandEnv(cmd.Args, cmd.Env) - - config.say("\n") // force a newline, there may have been loggy-gunk before this. - config.say("shortname: " + b.Name + "\n") - config.say("toolchain: " + config.Name + "\n") - s, rc = todo.Configurations[j].runBinary(dirs.wd, cmd, false) - } else { - // docker run --net=none -e GOROOT=... -w /src/github.com/minio/minio/cmd $D /testbin/cmd_Config.test -test.short -test.run=Nope -test.v -test.bench=Benchmark'(Get|Put|List)' - // TODO(jfaller): I don't think we need either of these "/" below, investigate... - - // TODO this all very undertested right now. - - bin := "/" + path.Join(dirs.testBinDir, testBinaryName) - wrappersAndBin = append(wrappersAndBin, bin) - - cmd := exec.Command("docker", "run", "--net=none", "-w", b.RunDir) - - for _, e := range runEnv { - cmd.Args = append(cmd.Args, "-e", e) - } - - cmd.Args = append(cmd.Args, "-e", "BENT_PROFILES="+path.Join(dirs.wd, config.thingBenchName("profiles"))) - if config.PgoGen != "" { - // We want to generate pprof file for using pgo - cmd.Args = append(cmd.Args, "-e", "BENT_PGO="+path.Join(dirs.wd, config.PgoGen)) - } - cmd.Args = append(cmd.Args, container) - cmd.Args = append(cmd.Args, wrappersAndBin...) - cmd.Args = append(cmd.Args, "-test.run="+b.Tests) - cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks) - - cmd.Args = append(cmd.Args, config.RunFlags...) - cmd.Args = append(cmd.Args, moreArgs...) - cmd.Args = sliceExpandEnv(cmd.Args, runEnv) - - config.say("\n") // force a newline, there may have been loggy-gunk before this. - config.say("shortname: " + b.Name + "\n") - config.say("toolchain: " + config.Name + "\n") - s, rc = todo.Configurations[j].runBinary(dirs.wd, cmd, false) - } if s != "" { fmt.Println(s) failures = append(failures, s) @@ -1103,6 +998,133 @@ } } +// benchOne runs a single benchmarks b in configuration c at iteration i, applying moreArgs to the run. +// it returns the output and the return code. +// +// if either the configuration or benchmark is disabled, return with empty output and no change (0) +// to the return code. +func benchOne(c *Configuration, b *Benchmark, i int, moreArgs []string) (s string, rc int) { + if c.Disabled || b.Disabled { + return + } + + root := c.Root + + testBinaryName := c.benchName(b) + + runEnv := []string{} + runEnv = append(runEnv, "BENT_CONFIG="+c.Name) + runEnv = append(runEnv, "BENT_BENCH="+b.Name) + runEnv = append(runEnv, "BENT_I="+strconv.FormatInt(int64(i), 10)) + runEnv = append(runEnv, "BENT_BINARY="+testBinaryName) + + wrapperPrefix := "/" + if b.NotSandboxed { + wrapperPrefix = dirs.wd + "/" + } + wrapperFor := func(s []string) string { + x := "" + if len(s) > 0 { + // If not an explicit path, then make it an explicit path + x = s[0] + if x[0] != '/' { + x = wrapperPrefix + x + } + } + return x + } + + crw := c.RunWrapper + + if c.PgoGen != "" { + // We want to generate pprof file for using pgo + crw = append(crw, wrapperFor([]string{"cpuprofile"})) + } + + configWrapper := wrapperFor(crw) + benchWrapper := wrapperFor(b.RunWrapper) + + var wrappersAndBin []string + + if configWrapper != "" { + wrappersAndBin = append(wrappersAndBin, configWrapper) + wrappersAndBin = append(wrappersAndBin, crw[1:]...) + } + if benchWrapper != "" { + wrappersAndBin = append(wrappersAndBin, benchWrapper) + wrappersAndBin = append(wrappersAndBin, b.RunWrapper[1:]...) + } + + if b.NotSandboxed { + bin := path.Join(dirs.wd, dirs.testBinDir, testBinaryName) + wrappersAndBin = append(wrappersAndBin, bin) + + cmd := exec.Command(wrappersAndBin[0], wrappersAndBin[1:]...) + cmd.Args = append(cmd.Args, "-test.run="+b.Tests) + cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks) + + cmd.Dir = b.RunDir + cmd.Env = DefaultEnv() + if root != "" { + cmd.Env = replaceEnv(cmd.Env, "GOROOT", root) + } + cmd.Env = append(cmd.Env, "BENT_DIR="+dirs.wd) + cmd.Env = append(cmd.Env, "BENT_PROFILES="+path.Join(dirs.wd, c.thingBenchName("profiles"))) + if c.PgoGen != "" { + // We want to generate pprof file for using pgo + cmd.Env = append(cmd.Env, "BENT_PGO="+path.Join(dirs.wd, c.PgoGen)) + } + + cmd.Env = append(cmd.Env, runEnv...) + cmd.Env = append(cmd.Env, sliceExpandEnv(c.RunEnv, cmd.Env)...) + + cmd.Args = append(cmd.Args, c.RunFlags...) + cmd.Args = append(cmd.Args, moreArgs...) + cmd.Args = sliceExpandEnv(cmd.Args, cmd.Env) + + c.say("\n") // force a newline, there may have been loggy-gunk before this. + c.say("shortname: " + b.Name + "\n") + c.say("toolchain: " + c.Name + "\n") + s, rc = c.runBinary(dirs.wd, cmd, false) + } else { + // docker run --net=none -e GOROOT=... -w /src/github.com/minio/minio/cmd $D /testbin/cmd_Config.test -test.short -test.run=Nope -test.v -test.bench=Benchmark'(Get|Put|List)' + // TODO(jfaller): I don't think we need either of these "/" below, investigate... + + // TODO this all very undertested right now. + + bin := "/" + path.Join(dirs.testBinDir, testBinaryName) + wrappersAndBin = append(wrappersAndBin, bin) + + cmd := exec.Command("docker", "run", "--net=none", "-w", b.RunDir) + + for _, e := range runEnv { + cmd.Args = append(cmd.Args, "-e", e) + } + + cmd.Args = append(cmd.Args, "-e", "BENT_PROFILES="+path.Join(dirs.wd, c.thingBenchName("profiles"))) + + if c.PgoGen != "" { + // We want to generate pprof file for using pgo + cmd.Args = append(cmd.Args, "-e", "BENT_PGO="+path.Join(dirs.wd, c.PgoGen)) + } + + cmd.Args = append(cmd.Args, container) + cmd.Args = append(cmd.Args, wrappersAndBin...) + cmd.Args = append(cmd.Args, "-test.run="+b.Tests) + cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks) + + cmd.Args = append(cmd.Args, c.RunFlags...) + cmd.Args = append(cmd.Args, moreArgs...) + cmd.Args = sliceExpandEnv(cmd.Args, runEnv) + + c.say("\n") // force a newline, there may have been loggy-gunk before this. + c.say("shortname: " + b.Name + "\n") + c.say("toolchain: " + c.Name + "\n") + s, rc = c.runBinary(dirs.wd, cmd, false) + } + return s, rc +} + var goMod = `module build go %s
diff --git a/cmd/bent/configs/configurations-pgo.toml b/cmd/bent/configs/configurations-pgo.toml index 93b9936..04aea58 100644 --- a/cmd/bent/configs/configurations-pgo.toml +++ b/cmd/bent/configs/configurations-pgo.toml
@@ -2,10 +2,9 @@ Name = "pgo-generate" Root = "$HOME/work/go/" PgoGen = "profiles" # Sub-directory where we put profiles -# Disabled = true + Disabled = true # Request this explicitly [[Configurations]] Name = "pgo-use" Root = "$HOME/work/go/" PgoUse = "profiles" # Sub-directory where we search profiles - Disabled = true
diff --git a/cmd/bent/configs/suites.toml b/cmd/bent/configs/suites.toml index a528766..826ec1c 100644 --- a/cmd/bent/configs/suites.toml +++ b/cmd/bent/configs/suites.toml
@@ -1,6 +1,12 @@ # A Suite is a short name, a repo, a version, and sometimes some related default values # for the benchmark specification files. + [[Suites]] + Name = "toml" + Repo = "github.com/BurntSushi/toml" + Version = "@v1.3.2" + ExtraFiles = ["_example"] + [[Suites]] Name = "dustin_broadcast" Repo = "github.com/dustin/go-broadcast"