cmd/bent: early check for missing commands Further address cherrymui's friction log. Also change default to NOT use Docker. Update scripts to remove -U flag. Change-Id: Ibb1e4ec0b127be265375e9bb61be5e740b8d3632 Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/354409 Trust: David Chase <drchase@google.com> Run-TryBot: David Chase <drchase@google.com> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Michael Pratt <mpratt@google.com> Reviewed-by: Cherry Mui <cherryyz@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go index 942b649..5eaa269 100644 --- a/cmd/bent/bent.go +++ b/cmd/bent/bent.go
@@ -75,7 +75,6 @@ var initialize = false var test = false var force = false -var noSandbox = false var requireSandbox = false var getOnly = false var runContainer = "" // if nonempty, skip builds and use existing named container (or binaries if -U ) @@ -144,13 +143,12 @@ flag.IntVar(&shuffle, "s", shuffle, "dimensionality of (build) shuffling (0-3), 0 = none, 1 = per-benchmark, configuration ordering, 2 = bench, config pairs, 3 = across repetitions.") flag.StringVar(&benchmarksString, "b", "", "comma-separated list of test/benchmark names (default is all)") - flag.StringVar(&benchFile, "B", benchFile, "name of file describing benchmarks") + flag.StringVar(&benchFile, "B", benchFile, "name of file containing benchmarks to run") flag.StringVar(&configurationsString, "c", "", "comma-separated list of test/benchmark configurations (default is all)") flag.StringVar(&confFile, "C", confFile, "name of file describing configurations") - flag.BoolVar(&noSandbox, "U", noSandbox, "run all commands unsandboxed") - flag.BoolVar(&requireSandbox, "S", requireSandbox, "exclude unsandboxable tests/benchmarks") + flag.BoolVar(&requireSandbox, "S", requireSandbox, "require Docker sandbox to run tests/benchmarks (& exclude unsandboxable tests/benchmarks)") flag.BoolVar(&getOnly, "g", getOnly, "get tests/benchmarks and dependencies, do not build or run") flag.StringVar(&runContainer, "r", runContainer, "skip get and build, go directly to run, using specified container (any non-empty string will do for unsandboxed execution)") @@ -173,37 +171,32 @@ ` %s obtains the benchmarks/tests listed in %s and compiles and runs them according to the flags and environment -variables supplied in %s. Specifying "-a" will pass "-a" to -test compilations, but normally this should not be needed -and only slows down builds; -a with a number that is not 1 -can be used for benchmarking builds of the tests themselves. -(Don't forget to specify "all=..." for GCFLAGS if you want +variables supplied in %s. + +Specifying "-a" will pass "-a" to test compilations, but normally this +should not be needed and only slows down builds; -a with a number +that is not 1 can be used for benchmarking builds of the tests +themselves. (Don't forget to specify "all=..." for GCFLAGS if you want those applied to the entire build.) Both of these files can be changed with the -B and -C flags; the full -suite of benchmarks in benchmarks-all.toml is somewhat time-consuming. +suite of benchmarks in benchmarks-all.toml is somewhat +time-consuming. Suites.toml contains the known benchmarks, their +versions, and certain necessary build or run flags. -Running with the -l flag will list all the available tests and benchmarks -for the given benchmark and configuration files. +Running with the -l flag will list all the available tests and +benchmarks for the given benchmark and configuration files. -By default the compiled tests are run in a docker container to reduce -the chances for accidents and mischief. -U requests running tests -unsandboxed, and -S limits the tests run to those that can be sandboxed -(some cannot be because of cross-compilation issues; this may imply no -change on platforms where the Docker container is not cross-compiled) +By default benchmarks are run, not tests. -T runs tests instead. -By default benchmarks are run, not tests. -T runs tests instead +To run tests or benchmnarks in a docker sandbox, specify -S; if the +host OS is not linux this will exclude some benchmarks that cannot be +cross-compiled. -This command expects to be run in a directory that does not contain -subdirectories "gopath/pkg" and "gopath/bin", because those subdirectories -may be created (and deleted) in the process of compiling the benchmarks. -The same is true of subdirectory "goroots". -It will also extensively modify subdirectory "gopath/src". - -All the test binaries will appear in the subdirectory 'testbin', -and test (benchmark) output will appear in the subdirectory 'bench' -with the suffix '.stdout'. The test output is grouped by configuration -to allow easy benchmark comparisons with benchstat. Other benchmarking +All the test binaries will appear in the subdirectory 'testbin', and +test (benchmark) output will appear in the subdirectory 'bench' with +the suffix '.stdout'. The test output is grouped by configuration to +allow easy benchmark comparisons with benchstat. Other benchmarking results will also appear in 'bench'. `, os.Args[0], benchFile, confFile) } @@ -216,6 +209,30 @@ os.Exit(1) } + // Fail early if either of these commands is missing. + _, errTime := exec.LookPath("/usr/bin/time") + _, errRsync := exec.LookPath("rsync") + if errTime != nil && errRsync != nil { + println("This program needs /usr/bin/time and rsync commands to run") + os.Exit(1) + } + if errRsync != nil { + println("This program needs the rsync command to run") + os.Exit(1) + } + if errTime != nil { + println("This program needs the /usr/bin/time command to run") + os.Exit(1) + } + + if requireSandbox { + _, errDocker := exec.LookPath("docker") + if errDocker != nil { + println("Sandboxing benchmarks requires the docker command") + os.Exit(1) + } + } + todo := &Todo{} blobB, err := ioutil.ReadFile(benchFile) if err != nil { @@ -355,7 +372,7 @@ todo.Benchmarks[i].Benchmarks = "none" } } - if noSandbox { + if !requireSandbox { todo.Benchmarks[i].NotSandboxed = true } if requireSandbox && todo.Benchmarks[i].NotSandboxed {
diff --git a/cmd/bent/scripts/cmpcl-phase.sh b/cmd/bent/scripts/cmpcl-phase.sh index 2e78a3e..4ce1112 100755 --- a/cmd/bent/scripts/cmpcl-phase.sh +++ b/cmd/bent/scripts/cmpcl-phase.sh
@@ -93,6 +93,6 @@ export STAMP cd "${ROOT}" -${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -c=Old-phase,New-phase -C=configurations-cmpjob.toml "$@" | tee phases.${STAMP}.log +${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -c=Old-phase,New-phase -C=configurations-cmpjob.toml "$@" | tee phases.${STAMP}.log phase-times > phases.${STAMP}.csv
diff --git a/cmd/bent/scripts/cmpcl.sh b/cmd/bent/scripts/cmpcl.sh index 3e7877e..be5cb73 100755 --- a/cmd/bent/scripts/cmpcl.sh +++ b/cmd/bent/scripts/cmpcl.sh
@@ -88,7 +88,7 @@ export newtag cd "${ROOT}" -${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@" +${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@" RUN=`tail -1 bentjobs.log | awk -c '{print $1}'` cd bench
diff --git a/cmd/bent/scripts/cmpjob.sh b/cmd/bent/scripts/cmpjob.sh index c5ac036..b8fc8be 100755 --- a/cmd/bent/scripts/cmpjob.sh +++ b/cmd/bent/scripts/cmpjob.sh
@@ -82,7 +82,7 @@ fi cd "${ROOT}" -${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@" +${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@" RUN=`tail -1 bentjobs.log | awk -c '{print $1}'` cd bench
diff --git a/cmd/bent/scripts/cronjob.sh b/cmd/bent/scripts/cronjob.sh index 8d45ca3..0cf275f 100755 --- a/cmd/bent/scripts/cronjob.sh +++ b/cmd/bent/scripts/cronjob.sh
@@ -70,7 +70,7 @@ cd "${ROOT}" # For arm64 big.little, might need to prefix with something like: # GOMAXPROCS=4 numactl -C 2-5 -- ... -${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@" +${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@" RUN=`tail -1 bentjobs.log | awk -c '{print $1}'` cd bench @@ -106,7 +106,7 @@ # Debugging build cd "${ROOT}" -${PERFLOCK} bent -U -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl "$@" +${PERFLOCK} bent -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl "$@" RUN=`tail -1 bentjobsNl.log | awk -c '{print $1}'` cd bench