cmd/bent: early check for missing commands
Further address cherrymui's friction log.
Also change default to NOT use Docker.
Update scripts to remove -U flag.
Change-Id: Ibb1e4ec0b127be265375e9bb61be5e740b8d3632
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/354409
Trust: David Chase <drchase@google.com>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index 942b649..5eaa269 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -75,7 +75,6 @@
var initialize = false
var test = false
var force = false
-var noSandbox = false
var requireSandbox = false
var getOnly = false
var runContainer = "" // if nonempty, skip builds and use existing named container (or binaries if -U )
@@ -144,13 +143,12 @@
flag.IntVar(&shuffle, "s", shuffle, "dimensionality of (build) shuffling (0-3), 0 = none, 1 = per-benchmark, configuration ordering, 2 = bench, config pairs, 3 = across repetitions.")
flag.StringVar(&benchmarksString, "b", "", "comma-separated list of test/benchmark names (default is all)")
- flag.StringVar(&benchFile, "B", benchFile, "name of file describing benchmarks")
+ flag.StringVar(&benchFile, "B", benchFile, "name of file containing benchmarks to run")
flag.StringVar(&configurationsString, "c", "", "comma-separated list of test/benchmark configurations (default is all)")
flag.StringVar(&confFile, "C", confFile, "name of file describing configurations")
- flag.BoolVar(&noSandbox, "U", noSandbox, "run all commands unsandboxed")
- flag.BoolVar(&requireSandbox, "S", requireSandbox, "exclude unsandboxable tests/benchmarks")
+ flag.BoolVar(&requireSandbox, "S", requireSandbox, "require Docker sandbox to run tests/benchmarks (& exclude unsandboxable tests/benchmarks)")
flag.BoolVar(&getOnly, "g", getOnly, "get tests/benchmarks and dependencies, do not build or run")
flag.StringVar(&runContainer, "r", runContainer, "skip get and build, go directly to run, using specified container (any non-empty string will do for unsandboxed execution)")
@@ -173,37 +171,32 @@
`
%s obtains the benchmarks/tests listed in %s and compiles
and runs them according to the flags and environment
-variables supplied in %s. Specifying "-a" will pass "-a" to
-test compilations, but normally this should not be needed
-and only slows down builds; -a with a number that is not 1
-can be used for benchmarking builds of the tests themselves.
-(Don't forget to specify "all=..." for GCFLAGS if you want
+variables supplied in %s.
+
+Specifying "-a" will pass "-a" to test compilations, but normally this
+should not be needed and only slows down builds; -a with a number
+that is not 1 can be used for benchmarking builds of the tests
+themselves. (Don't forget to specify "all=..." for GCFLAGS if you want
those applied to the entire build.)
Both of these files can be changed with the -B and -C flags; the full
-suite of benchmarks in benchmarks-all.toml is somewhat time-consuming.
+suite of benchmarks in benchmarks-all.toml is somewhat
+time-consuming. Suites.toml contains the known benchmarks, their
+versions, and certain necessary build or run flags.
-Running with the -l flag will list all the available tests and benchmarks
-for the given benchmark and configuration files.
+Running with the -l flag will list all the available tests and
+benchmarks for the given benchmark and configuration files.
-By default the compiled tests are run in a docker container to reduce
-the chances for accidents and mischief. -U requests running tests
-unsandboxed, and -S limits the tests run to those that can be sandboxed
-(some cannot be because of cross-compilation issues; this may imply no
-change on platforms where the Docker container is not cross-compiled)
+By default benchmarks are run, not tests. -T runs tests instead.
-By default benchmarks are run, not tests. -T runs tests instead
+To run tests or benchmnarks in a docker sandbox, specify -S; if the
+host OS is not linux this will exclude some benchmarks that cannot be
+cross-compiled.
-This command expects to be run in a directory that does not contain
-subdirectories "gopath/pkg" and "gopath/bin", because those subdirectories
-may be created (and deleted) in the process of compiling the benchmarks.
-The same is true of subdirectory "goroots".
-It will also extensively modify subdirectory "gopath/src".
-
-All the test binaries will appear in the subdirectory 'testbin',
-and test (benchmark) output will appear in the subdirectory 'bench'
-with the suffix '.stdout'. The test output is grouped by configuration
-to allow easy benchmark comparisons with benchstat. Other benchmarking
+All the test binaries will appear in the subdirectory 'testbin', and
+test (benchmark) output will appear in the subdirectory 'bench' with
+the suffix '.stdout'. The test output is grouped by configuration to
+allow easy benchmark comparisons with benchstat. Other benchmarking
results will also appear in 'bench'.
`, os.Args[0], benchFile, confFile)
}
@@ -216,6 +209,30 @@
os.Exit(1)
}
+ // Fail early if either of these commands is missing.
+ _, errTime := exec.LookPath("/usr/bin/time")
+ _, errRsync := exec.LookPath("rsync")
+ if errTime != nil && errRsync != nil {
+ println("This program needs /usr/bin/time and rsync commands to run")
+ os.Exit(1)
+ }
+ if errRsync != nil {
+ println("This program needs the rsync command to run")
+ os.Exit(1)
+ }
+ if errTime != nil {
+ println("This program needs the /usr/bin/time command to run")
+ os.Exit(1)
+ }
+
+ if requireSandbox {
+ _, errDocker := exec.LookPath("docker")
+ if errDocker != nil {
+ println("Sandboxing benchmarks requires the docker command")
+ os.Exit(1)
+ }
+ }
+
todo := &Todo{}
blobB, err := ioutil.ReadFile(benchFile)
if err != nil {
@@ -355,7 +372,7 @@
todo.Benchmarks[i].Benchmarks = "none"
}
}
- if noSandbox {
+ if !requireSandbox {
todo.Benchmarks[i].NotSandboxed = true
}
if requireSandbox && todo.Benchmarks[i].NotSandboxed {
diff --git a/cmd/bent/scripts/cmpcl-phase.sh b/cmd/bent/scripts/cmpcl-phase.sh
index 2e78a3e..4ce1112 100755
--- a/cmd/bent/scripts/cmpcl-phase.sh
+++ b/cmd/bent/scripts/cmpcl-phase.sh
@@ -93,6 +93,6 @@
export STAMP
cd "${ROOT}"
-${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -c=Old-phase,New-phase -C=configurations-cmpjob.toml "$@" | tee phases.${STAMP}.log
+${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -c=Old-phase,New-phase -C=configurations-cmpjob.toml "$@" | tee phases.${STAMP}.log
phase-times > phases.${STAMP}.csv
diff --git a/cmd/bent/scripts/cmpcl.sh b/cmd/bent/scripts/cmpcl.sh
index 3e7877e..be5cb73 100755
--- a/cmd/bent/scripts/cmpcl.sh
+++ b/cmd/bent/scripts/cmpcl.sh
@@ -88,7 +88,7 @@
export newtag
cd "${ROOT}"
-${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@"
+${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@"
RUN=`tail -1 bentjobs.log | awk -c '{print $1}'`
cd bench
diff --git a/cmd/bent/scripts/cmpjob.sh b/cmd/bent/scripts/cmpjob.sh
index c5ac036..b8fc8be 100755
--- a/cmd/bent/scripts/cmpjob.sh
+++ b/cmd/bent/scripts/cmpjob.sh
@@ -82,7 +82,7 @@
fi
cd "${ROOT}"
-${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@"
+${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cmpjob.toml "$@"
RUN=`tail -1 bentjobs.log | awk -c '{print $1}'`
cd bench
diff --git a/cmd/bent/scripts/cronjob.sh b/cmd/bent/scripts/cronjob.sh
index 8d45ca3..0cf275f 100755
--- a/cmd/bent/scripts/cronjob.sh
+++ b/cmd/bent/scripts/cronjob.sh
@@ -70,7 +70,7 @@
cd "${ROOT}"
# For arm64 big.little, might need to prefix with something like:
# GOMAXPROCS=4 numactl -C 2-5 -- ...
-${PERFLOCK} bent -U -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@"
+${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@"
RUN=`tail -1 bentjobs.log | awk -c '{print $1}'`
cd bench
@@ -106,7 +106,7 @@
# Debugging build
cd "${ROOT}"
-${PERFLOCK} bent -U -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl "$@"
+${PERFLOCK} bent -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl "$@"
RUN=`tail -1 bentjobsNl.log | awk -c '{print $1}'`
cd bench
