cmd/bent: change to use module-mode for building
also tweaks for performance dashboard
also build tags to turn off broken unsafe code in some benchmarks
also corrected profiling for new run directory
Change-Id: I27f484e52de4ce55f90bb3f122be72127a3181b4
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/354410
Trust: David Chase <drchase@google.com>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index 5eaa269..4081800 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -20,7 +20,6 @@
"os"
"os/exec"
"path"
- "path/filepath"
"runtime"
"strconv"
"strings"
@@ -44,8 +43,9 @@
BuildFlags []string // Flags for building test (e.g., -tags purego)
RunWrapper []string // (Inner) Command and args to precede whatever the operation is; may fail in the sandbox.
// e.g. benchmark may run as ConfigWrapper ConfigArg BenchWrapper BenchArg ActualBenchmark
- NotSandboxed bool // True if this benchmark cannot or should not be run in a container.
- Disabled bool // True if this benchmark is temporarily disabled.
+ NotSandboxed bool // True if this benchmark cannot or should not be run in a container.
+ Disabled bool // True if this benchmark is temporarily disabled.
+ RunDir string // Parent directory of testdata.
}
type Todo struct {
@@ -110,26 +110,10 @@
var runstamp = strings.Replace(strings.Replace(time.Now().Format("2006-01-02T15:04:05"), "-", "", -1), ":", "", -1)
func cleanup(gopath string) {
- pkg, bin := path.Join(gopath, "pkg"), path.Join(gopath, "bin")
+ bin := path.Join(gopath, "bin")
if verbose > 0 {
- fmt.Printf("chmod -R u+w %s\n", pkg)
+ fmt.Printf("rm -rf %s\n", bin)
}
- // Necessary to make directories writeable with new module stuff.
- filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error {
- if path != "" && info != nil {
- if mode := info.Mode(); 0 == mode&os.ModeSymlink {
- err := os.Chmod(path, 0200|mode)
- if err != nil {
- panic(err)
- }
- }
- }
- return nil
- })
- if verbose > 0 {
- fmt.Printf("rm -rf %s %s\n", pkg, bin)
- }
- os.RemoveAll(pkg)
os.RemoveAll(bin)
}
@@ -203,36 +187,44 @@
flag.Parse()
- // Make sure our filesystem is in good shape.
- if err := checkAndSetUpFileSystem(initialize); err != nil {
- fmt.Printf("%v", err)
- os.Exit(1)
- }
-
// Fail early if either of these commands is missing.
_, errTime := exec.LookPath("/usr/bin/time")
_, errRsync := exec.LookPath("rsync")
if errTime != nil && errRsync != nil {
- println("This program needs /usr/bin/time and rsync commands to run")
+ fmt.Println("This program needs /usr/bin/time and rsync commands to run")
os.Exit(1)
}
if errRsync != nil {
- println("This program needs the rsync command to run")
+ fmt.Println("This program needs the rsync command to run")
os.Exit(1)
}
if errTime != nil {
- println("This program needs the /usr/bin/time command to run")
+ fmt.Println("This program needs the /usr/bin/time command to run")
os.Exit(1)
}
if requireSandbox {
_, errDocker := exec.LookPath("docker")
if errDocker != nil {
- println("Sandboxing benchmarks requires the docker command")
+ fmt.Println("Sandboxing benchmarks requires the docker command")
os.Exit(1)
}
}
+ // Make sure our filesystem is in good shape.
+ if err := checkAndSetUpFileSystem(initialize); err != nil {
+ fmt.Printf("%v", err)
+ os.Exit(1)
+ }
+
+ var err error
+ // Create any directories we need.
+ dirs, err = createDirectories(0775)
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+
todo := &Todo{}
blobB, err := ioutil.ReadFile(benchFile)
if err != nil {
@@ -251,16 +243,6 @@
os.Exit(1)
}
- // Create any directories we need.
- dirs, err := createDirectories(0775)
- if err != nil {
- fmt.Println(err)
- os.Exit(1)
- }
- for i := range todo.Configurations {
- todo.Configurations[i].dirs = dirs
- }
-
var moreArgs []string
if flag.NArg() > 0 {
for i, arg := range flag.Args() {
@@ -488,6 +470,26 @@
fmt.Print("Go getting")
}
+ goDotMod := path.Join(dirs.build, "go.mod")
+
+ if _, err := os.Stat(goDotMod); err != nil { // if error assume go.mod does not exist
+ cmd := exec.Command("go", "mod", "init", "build")
+ cmd.Env = defaultEnv
+ cmd.Dir = dirs.build
+
+ if verbose > 0 {
+ fmt.Println(asCommandLine(dirs.wd, cmd))
+ } else {
+ fmt.Print(".")
+ }
+ _, err := cmd.Output()
+ if err != nil {
+ ee := err.(*exec.ExitError)
+ fmt.Printf("There was an error running 'go mod init', stderr = %s", ee.Stderr)
+ os.Exit(2)
+ }
+ }
+
// Obtain (go get -d -t -v bench.Repo) all benchmarks, once, populating src
for i, bench := range todo.Benchmarks {
if bench.Disabled {
@@ -497,6 +499,7 @@
cmd := exec.Command("go", "get", "-d", "-t", "-v", bench.Repo)
cmd.Env = getBuildEnv
+ cmd.Dir = dirs.build
if !bench.NotSandboxed { // Do this so that OS-dependent dependencies are done correctly.
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
@@ -516,54 +519,6 @@
continue
}
- // Ensure testdir exists -- if modules are enabled, it does not.
- // This involves invoking git to make it appear.
- testdir := path.Join(dirs.gopath, "src", bench.Repo)
- _, terr := os.Stat(testdir)
- if terr != nil { // Assume missing directory is the cause of the error.
- parts := strings.Split(bench.Repo, "/")
- root := parts[0]
- repoAt := pathLengths[root] - 1
- if repoAt < 1 || repoAt >= len(parts) {
- s := fmt.Sprintf("repoAt=%d was not a valid index for %v", repoAt, parts)
- fmt.Println(s + "DISABLING benchmark " + bench.Name)
- getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
- todo.Benchmarks[i].Disabled = true
- continue
- }
- dirToMake := path.Join(dirs.gopath, "src", path.Join(parts[:repoAt]...))
- repoToGet := path.Join(parts[:repoAt+1]...)
- if verbose > 0 {
- fmt.Printf("mkdir -p %s\n", dirToMake)
- }
- err := os.MkdirAll(dirToMake, 0777)
- if err != nil {
- s := fmt.Sprintf("could not os.MkdirAll(%s), err = %v", dirToMake, err)
- fmt.Println(s + "DISABLING benchmark " + bench.Name)
- getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
- todo.Benchmarks[i].Disabled = true
- continue
- }
-
- cmd = exec.Command("git", "clone", "https://"+repoToGet)
- cmd.Env = defaultEnv
- cmd.Dir = dirToMake
- if verbose > 0 {
- fmt.Println(asCommandLine(dirs.wd, cmd))
- } else {
- fmt.Print(".")
- }
- _, err = cmd.Output()
- if err != nil {
- ee := err.(*exec.ExitError)
- s := fmt.Sprintf("There was an error running 'git clone', stderr = %s", ee.Stderr)
- fmt.Println(s + "DISABLING benchmark " + bench.Name)
- getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
- todo.Benchmarks[i].Disabled = true
- continue
- }
- }
-
needSandbox = !bench.NotSandboxed || needSandbox
needNotSandbox = bench.NotSandboxed || needNotSandbox
}
@@ -807,6 +762,41 @@
}
}
+ // Initialize RunDir for benchmarks.
+ for i, bench := range todo.Benchmarks {
+ if bench.Disabled {
+ continue
+ }
+ // Obtain directory containing testdata, if any:
+ // Capture output of "go list -f {{.Dir}} $PKG"
+
+ cmd := exec.Command("go", "list", "-f", "{{.Dir}}", bench.Repo)
+ cmd.Env = defaultEnv
+ cmd.Dir = dirs.build
+
+ if verbose > 0 {
+ fmt.Println(asCommandLine(dirs.wd, cmd))
+ } else {
+ fmt.Print(".")
+ }
+ out, err := cmd.Output()
+ if err != nil {
+ s := fmt.Sprintf(`could not go list -f {{.Dir}} %s, err=%v`, bench.Repo, err)
+ fmt.Println(s + "DISABLING benchmark " + bench.Name)
+ getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
+ todo.Benchmarks[i].Disabled = true
+ continue
+ } else if verbose > 0 {
+ fmt.Printf("# Rundir=%s\n", string(out))
+ }
+ rundir := strings.TrimSpace(string(out))
+ if !bench.NotSandboxed {
+ // if sandboxed, strip cwd from prefix of rundir.
+ rundir = rundir[len(dirs.wd):]
+ }
+ todo.Benchmarks[i].RunDir = rundir
+ }
+
var failures []string
// If there's a bad error running one of the benchmarks, report what we've got, please.
@@ -887,7 +877,6 @@
}
if b.NotSandboxed {
- testdir := path.Join(dirs.gopath, "src", b.Repo)
bin := path.Join(dirs.wd, dirs.testBinDir, testBinaryName)
wrappersAndBin = append(wrappersAndBin, bin)
@@ -895,31 +884,33 @@
cmd.Args = append(cmd.Args, "-test.run="+b.Tests)
cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks)
- cmd.Dir = testdir
+ cmd.Dir = b.RunDir
cmd.Env = defaultEnv
if root != "" {
cmd.Env = replaceEnv(cmd.Env, "GOROOT", root)
}
cmd.Env = replaceEnvs(cmd.Env, config.RunEnv)
cmd.Env = append(cmd.Env, "BENT_DIR="+dirs.wd)
+ cmd.Env = append(cmd.Env, "BENT_PROFILES="+path.Join(dirs.wd, config.thingBenchName("profiles")))
cmd.Env = append(cmd.Env, "BENT_BINARY="+testBinaryName)
cmd.Env = append(cmd.Env, "BENT_I="+strconv.FormatInt(int64(i), 10))
cmd.Args = append(cmd.Args, config.RunFlags...)
cmd.Args = append(cmd.Args, moreArgs...)
+
+ config.say("shortname: " + b.Name + "\n")
s, rc = todo.Configurations[j].runBinary(dirs.wd, cmd, false)
} else {
// docker run --net=none -e GOROOT=... -w /src/github.com/minio/minio/cmd $D /testbin/cmd_Config.test -test.short -test.run=Nope -test.v -test.bench=Benchmark'(Get|Put|List)'
// TODO(jfaller): I don't think we need either of these "/" below, investigate...
- testdir := "/" + path.Join("gopath", "src", b.Repo)
bin := "/" + path.Join(dirs.testBinDir, testBinaryName)
wrappersAndBin = append(wrappersAndBin, bin)
- cmd := exec.Command("docker", "run", "--net=none",
- "-w", testdir)
+ cmd := exec.Command("docker", "run", "--net=none", "-w", b.RunDir)
for _, e := range config.RunEnv {
cmd.Args = append(cmd.Args, "-e", e)
}
cmd.Args = append(cmd.Args, "-e", "BENT_DIR=/") // TODO this is not going to work well
+ cmd.Args = append(cmd.Args, "-e", "BENT_PROFILES="+path.Join(dirs.wd, config.thingBenchName("profiles")))
cmd.Args = append(cmd.Args, "-e", "BENT_BINARY="+testBinaryName)
cmd.Args = append(cmd.Args, "-e", "BENT_I="+strconv.FormatInt(int64(i), 10))
cmd.Args = append(cmd.Args, container)
@@ -928,6 +919,7 @@
cmd.Args = append(cmd.Args, "-test.bench="+b.Benchmarks)
cmd.Args = append(cmd.Args, config.RunFlags...)
cmd.Args = append(cmd.Args, moreArgs...)
+ config.say("shortname: " + b.Name + "\n")
s, rc = todo.Configurations[j].runBinary(dirs.wd, cmd, false)
}
if s != "" {
@@ -990,16 +982,7 @@
// To avoid bad surprises, look for pkg and bin, if they exist, refuse to run
_, derr := os.Stat("Dockerfile")
_, perr := os.Stat(path.Join("gopath", "pkg"))
- _, berr := os.Stat(path.Join("gopath", "bin"))
- _, serr := os.Stat(path.Join("gopath", "src")) // existence of src prevents initialization of Dockerfile
- if perr == nil || berr == nil {
- if !force {
- return errors.New("Building/running tests will trash gopath/pkg and gopath/bin, please remove, rename or run in another directory, or use -f to force.\n")
- }
- fmt.Printf("Building/running tests will trash gopath/pkg and gopath/bin, but force, so removing.\n")
- cleanup("gopath")
- }
if derr != nil && !shouldInit {
// Missing Dockerfile
return errors.New("Missing 'Dockerfile', please rerun with -I (initialize) flag if you intend to use this directory.\n")
@@ -1011,8 +994,8 @@
// Initialize the directory, copying in default benchmarks and sample configurations, and creating a Dockerfile
if shouldInit {
- if serr == nil {
- fmt.Printf("It looks like you've already initialized this directory, remove ./gopath if you want to reinit.\n")
+ if perr == nil {
+ fmt.Printf("It looks like you've already initialized this directory, remove ./gopath/pkg if you want to reinit.\n")
os.Exit(1)
}
for _, s := range copyExes {
@@ -1062,7 +1045,7 @@
}
type directories struct {
- wd, gopath, goroots, testBinDir, benchDir string
+ wd, gopath, goroots, build, testBinDir, benchDir string
}
// createDirectories creates all the directories we need.
@@ -1076,10 +1059,11 @@
wd: cwd,
gopath: path.Join(cwd, "gopath"),
goroots: path.Join(cwd, "goroots"),
+ build: path.Join(cwd, "build"),
testBinDir: "testbin",
benchDir: "bench",
}
- for _, d := range []string{dirs.gopath, dirs.goroots, path.Join(cwd, dirs.testBinDir), path.Join(cwd, dirs.benchDir)} {
+ for _, d := range []string{dirs.gopath, dirs.goroots, dirs.build, path.Join(cwd, dirs.testBinDir), path.Join(cwd, dirs.benchDir)} {
if err := os.Mkdir(d, mode); err != nil && !errors.Is(err, fs.ErrExist) {
return nil, fmt.Errorf("error creating %v: %v", d, err)
}
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml
index 2ce36be..6b79324 100644
--- a/cmd/bent/configs/benchmarks-50.toml
+++ b/cmd/bent/configs/benchmarks-50.toml
@@ -114,16 +114,19 @@
[[Benchmarks]]
Name = "gonum_blas_native"
Repo = "gonum.org/v1/gonum/blas/gonum"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(DasumMediumUnitaryInc|Dnrm2MediumPosInc)" # not all benchmarks
[[Benchmarks]]
Name = "gonum_lapack_native"
Repo = "gonum.org/v1/gonum/lapack/gonum"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkDgeev/Circulant10"
[[Benchmarks]]
Name = "gonum_mat"
Repo = "gonum.org/v1/gonum/mat"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(MulWorkspaceDense1000Hundredth|ScaleVec10000Inc20)"
[[Benchmarks]]
@@ -156,21 +159,25 @@
[[Benchmarks]]
Name = "gonum_topo"
Repo = "gonum.org/v1/gonum/graph/topo/"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(TarjanSCCGnp_1000_half|TarjanSCCGnp_10_tenth)"
[[Benchmarks]]
Name = "gonum_path"
Repo = "gonum.org/v1/gonum/graph/path/"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(AStarUndirectedmallWorld_10_2_2_2_Heur|Dominators/nested_if_n256)"
[[Benchmarks]]
Name = "gonum_community"
Repo = "gonum.org/v1/gonum/graph/community/"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkLouvainDirectedMultiplex"
[[Benchmarks]]
Name = "gonum_traverse"
Repo = "gonum.org/v1/gonum/graph/traverse/"
+ BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkWalkAllBreadthFirstGnp_(10|1000)_tenth" # more difference by size than anything else
[[Benchmarks]]
diff --git a/cmd/bent/configuration.go b/cmd/bent/configuration.go
index a8fdf1a..89f1f25 100644
--- a/cmd/bent/configuration.go
+++ b/cmd/bent/configuration.go
@@ -37,10 +37,11 @@
Disabled bool // True if this configuration is temporarily disabled
buildStats []BenchStat
benchWriter *os.File
- rootCopy string // The contents of GOROOT are copied here to allow benchmarking of just the test compilation.
- dirs *directories // Test configuration
+ rootCopy string // The contents of GOROOT are copied here to allow benchmarking of just the test compilation.
}
+var dirs *directories // constant across all configurations, useful in other contexts.
+
func (c *Configuration) buildBenchName() string {
return c.thingBenchName("build")
}
@@ -49,7 +50,7 @@
if len(suffix) != 0 {
suffix = path.Base(suffix)
}
- return path.Join(c.dirs.benchDir, runstamp+"."+c.Name+"."+suffix)
+ return path.Join(dirs.benchDir, runstamp+"."+c.Name+"."+suffix)
}
func (c *Configuration) benchName(b *Benchmark) string {
@@ -119,7 +120,7 @@
continue
}
testBinaryName := config.benchName(b)
- c := exec.Command(cmd, path.Join(cwd, config.dirs.testBinDir, testBinaryName), b.Name)
+ c := exec.Command(cmd, path.Join(cwd, dirs.testBinDir, testBinaryName), b.Name)
c.Env = defaultEnv
if !b.NotSandboxed {
@@ -173,6 +174,8 @@
// Prefix with time for build benchmarking:
cmd := exec.Command("/usr/bin/time", "-p", gocmd, "test", "-vet=off", "-c")
+ compileTo := path.Join(dirs.wd, dirs.testBinDir, config.benchName(bench))
+ cmd.Args = append(cmd.Args, "-o", compileTo)
cmd.Args = append(cmd.Args, bench.BuildFlags...)
// Do not normally need -a because cache was emptied first and std was -a installed with these flags.
// But for -a=1, do it anyway
@@ -183,8 +186,8 @@
if config.GcFlags != "" {
cmd.Args = append(cmd.Args, "-gcflags="+config.GcFlags)
}
- cmd.Args = append(cmd.Args, ".")
- cmd.Dir = path.Join(gopath, "src", bench.Repo)
+ cmd.Args = append(cmd.Args, bench.Repo)
+ cmd.Dir = dirs.build // use module-mode
cmd.Env = defaultEnv
if !bench.NotSandboxed {
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
@@ -251,18 +254,8 @@
f.Sync()
f.Close()
- // Move generated binary to well-known place.
- from := path.Join(cmd.Dir, bench.testBinaryName())
- to := path.Join(config.dirs.wd, config.dirs.testBinDir, config.benchName(bench))
- err = os.Rename(from, to)
- if err != nil {
- fmt.Printf("There was an error renaming %s to %s, %v\n", from, to, err)
- cleanup(gopath)
- os.Exit(1)
- }
// Trim /usr/bin/time info from soutput, it's ugly
if verbose > 0 {
- fmt.Println("mv " + from + " " + to + "")
i := strings.LastIndex(soutput, "real")
if i >= 0 {
soutput = soutput[:i]
@@ -278,6 +271,17 @@
return ""
}
+// say writes s to c's benchmark output file
+func (c *Configuration) say(s string) {
+ b := []byte(s)
+ nw, err := c.benchWriter.Write(b)
+ if err != nil {
+ fmt.Printf("Error writing, err = %v, nwritten = %d, nrequested = %d\n", err, nw, len(b))
+ }
+ c.benchWriter.Sync()
+ fmt.Print(string(b))
+}
+
// runBinary runs cmd and displays the output.
// If the command returns an error, returns an error string.
func (c *Configuration) runBinary(cwd string, cmd *exec.Cmd, printWorkingDot bool) (string, int) {
diff --git a/cmd/bent/scripts/cpuprofile b/cmd/bent/scripts/cpuprofile
index c6b3d0a..111653f 100755
--- a/cmd/bent/scripts/cpuprofile
+++ b/cmd/bent/scripts/cpuprofile
@@ -1,8 +1,9 @@
#!/bin/bash
# Run args as command, but run cpuprofile and then pprof to capture test cpuprofile output
-pf="${BENT_BINARY}_${BENT_I}.prof"
+pf="${BENT_PROFILES}/${BENT_BINARY}_${BENT_I}.prof"
+mkdir -p ${BENT_PROFILES}
"$@" -test.cpuprofile="$pf"
-echo cpuprofile in `pwd`/"$pf"
+echo cpuprofile in "$pf"
if [[ x`which pprof` == x"" ]] ; then
go tool pprof -text -flat -nodecount=20 "$pf"
else
diff --git a/cmd/bent/scripts/memprofile b/cmd/bent/scripts/memprofile
index a8207a1..b459da5 100755
--- a/cmd/bent/scripts/memprofile
+++ b/cmd/bent/scripts/memprofile
@@ -1,6 +1,8 @@
#!/bin/bash
# Run args as command, but run memprofile and then pprof to capture test memprofile output
-mpf="${BENT_BINARY}_${BENT_I}.mprof"
+mpf="${BENT_PROFILES}/${BENT_BINARY}_${BENT_I}.mprof"
+mkdir -p ${BENT_PROFILES}
+
"$@" -test.memprofile="$mpf"
-echo memprofile in `pwd`/"$mpf"
+echo memprofile in "$mpf"
go tool pprof --alloc_space --text --cum --nodecount=20 "$mpf"
