cmd/bent: add version tags; benchmark curation and repair
Could be split into two CLs (.go files and .toml files)
but for testing this is good enough.
Change-Id: I2c1ee06a4917c5edc22c1e009a855dc4d4436061
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/354634
Trust: David Chase <drchase@google.com>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index 01e6ec1..f498b06 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -46,6 +46,8 @@
NotSandboxed bool // True if this benchmark cannot or should not be run in a container.
Disabled bool // True if this benchmark is temporarily disabled.
RunDir string // Parent directory of testdata.
+ BuildDir string // Location of go.mod for this benchmark; download here, go test -c here.
+ Version string // To pin a benchmark at a version.
}
type Todo struct {
@@ -210,7 +212,7 @@
var err error
// Create any directories we need.
- dirs, err = createDirectories(0775)
+ dirs, err = createDirectories()
if err != nil {
fmt.Println(err)
os.Exit(1)
@@ -331,6 +333,9 @@
bench.Repo = bench.Repo[:len(bench.Repo)-1]
todo.Benchmarks[i].Repo = bench.Repo
}
+ if "" == bench.Version {
+ todo.Benchmarks[i].Version = "@latest"
+ }
if "" == bench.Tests || !test {
if !test {
todo.Benchmarks[i].Tests = "none"
@@ -378,7 +383,7 @@
if list {
fmt.Println("Benchmarks:")
for _, x := range todo.Benchmarks {
- s := x.Name + " (repo=" + x.Repo + ")"
+ s := x.Name + " (repo=" + x.Repo + x.Version + ")"
if x.Disabled {
s += " (disabled)"
}
@@ -456,41 +461,58 @@
buildCount = 1
}
+ for i := range todo.Benchmarks {
+ bench := &todo.Benchmarks[i]
+
+ if bench.Disabled {
+ continue
+ }
+
+ // Use a separate build directory and go.mod for each benchmark, otherwise there can be conflicts.
+ // Initialize before building because this information tells where to run the test, also.
+ bench.BuildDir = path.Join(dirs.build, bench.Name)
+ }
+
if runContainer == "" { // If not reusing binaries/container...
if verbose == 0 {
fmt.Print("Go getting")
}
- goDotMod := path.Join(dirs.build, "go.mod")
-
- if _, err := os.Stat(goDotMod); err != nil { // if error assume go.mod does not exist
- cmd := exec.Command("go", "mod", "init", "build")
- cmd.Env = defaultEnv
- cmd.Dir = dirs.build
-
- if verbose > 0 {
- fmt.Println(asCommandLine(dirs.wd, cmd))
- } else {
- fmt.Print(".")
- }
- _, err := cmd.Output()
- if err != nil {
- ee := err.(*exec.ExitError)
- fmt.Printf("There was an error running 'go mod init', stderr = %s", ee.Stderr)
- os.Exit(2)
- }
- }
-
// Obtain (go get -d -t -v bench.Repo) all benchmarks, once, populating src
- for i, bench := range todo.Benchmarks {
+ for i := range todo.Benchmarks {
+ bench := &todo.Benchmarks[i]
+
if bench.Disabled {
continue
}
- getBuildEnv := replaceEnvs(defaultEnv, bench.GcEnv)
- cmd := exec.Command("go", "get", "-d", "-t", "-v", bench.Repo)
- cmd.Env = getBuildEnv
- cmd.Dir = dirs.build
+ // Use a separate go.mod for each benchmark, otherwise there can be conflicts.
+ if err := mkdirAsNeeded(bench.BuildDir); err != nil {
+ fmt.Printf("Couldn't create build subdirectory %s, error=%v", bench.BuildDir, err)
+ os.Exit(2)
+ }
+ goDotMod := path.Join(bench.BuildDir, "go.mod")
+ if _, err := os.Stat(goDotMod); err != nil { // if error assume go.mod does not exist
+ cmd := exec.Command("go", "mod", "init", "build")
+ cmd.Env = defaultEnv
+ cmd.Dir = bench.BuildDir
+
+ if verbose > 0 {
+ fmt.Println(asCommandLine(dirs.wd, cmd))
+ } else {
+ fmt.Print(".")
+ }
+ _, err := cmd.Output()
+ if err != nil {
+ ee := err.(*exec.ExitError)
+ fmt.Printf("There was an error running 'go mod init', stderr = %s", ee.Stderr)
+ os.Exit(2)
+ }
+ }
+
+ cmd := exec.Command("go", "get", "-d", "-t", "-v", bench.Repo+bench.Version)
+ cmd.Env = replaceEnvs(defaultEnv, bench.GcEnv)
+ cmd.Dir = bench.BuildDir
if !bench.NotSandboxed { // Do this so that OS-dependent dependencies are done correctly.
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
@@ -768,7 +790,7 @@
cmd := exec.Command("go", "list", "-f", "{{.Dir}}", bench.Repo)
cmd.Env = defaultEnv
- cmd.Dir = dirs.build
+ cmd.Dir = bench.BuildDir
if verbose > 0 {
fmt.Println(asCommandLine(dirs.wd, cmd))
@@ -1045,7 +1067,7 @@
}
// createDirectories creates all the directories we need.
-func createDirectories(mode fs.FileMode) (*directories, error) {
+func createDirectories() (*directories, error) {
cwd, err := os.Getwd()
if err != nil {
fmt.Printf("Could not get current working directory %v\n", err)
@@ -1060,13 +1082,21 @@
benchDir: "bench",
}
for _, d := range []string{dirs.gopath, dirs.goroots, dirs.build, path.Join(cwd, dirs.testBinDir), path.Join(cwd, dirs.benchDir)} {
- if err := os.Mkdir(d, mode); err != nil && !errors.Is(err, fs.ErrExist) {
+ if err := mkdirAsNeeded(d); err != nil {
return nil, fmt.Errorf("error creating %v: %v", d, err)
}
}
return dirs, nil
}
+// mkdirAsNeeded creates directory d with mode 0775 if it does not exist already.
+func mkdirAsNeeded(d string) error {
+ if err := os.Mkdir(d, 0775); err != nil && !errors.Is(err, fs.ErrExist) {
+ return fmt.Errorf("error creating %v: %v", d, err)
+ }
+ return nil
+}
+
// testBinaryName returns the name of the binary produced by "go test -c"
func (b *Benchmark) testBinaryName() string {
return b.Repo[strings.LastIndex(b.Repo, "/")+1:] + ".test"
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml
index 9c02e07..375e65f 100644
--- a/cmd/bent/configs/benchmarks-50.toml
+++ b/cmd/bent/configs/benchmarks-50.toml
@@ -5,32 +5,24 @@
[[Benchmarks]]
Name = "klauspost"
Repo = "github.com/dr2chase/benchmarks/klauspost"
- Tests = "" # Don't run these tests; they hang, etc.
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "minio"
Repo = "github.com/minio/minio/cmd"
Tests = "none" # Don't run these tests; they hang, etc.
- Benchmarks = "BenchmarkGetObject5MbFS" # not Get|Put|List
+ Benchmarks = "Benchmark(MarshalMsgdataUsageCacheInfo|DecodehealingTracker|DataUpdateTracker|AppendMsg(ReplicateDecision|ResyncTargetsInfo))"
GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
- Name = "hugo_hugolib_sitebuilding"
- Repo = "github.com/gohugoio/hugo/hugolib"
- Benchmarks = "BenchmarkSiteBuilding/YAML,num_pages=10,num_tags=10,tags_per_page=20,shortcodes,render-12"
- Disabled = true # Broken as of 2019-04-10
-
-[[Benchmarks]]
Name = "hugo_hugolib"
Repo = "github.com/gohugoio/hugo/hugolib"
- Benchmarks = "BenchmarkParsePage"
- Disabled = true # Broken as of 2019-04-10
+ Benchmarks = "Benchmark(ResourceChainPostProcess|ReplaceShortcodeTokens|MergeByLanguage|GetPage$|(Cascade/langs-1-8))"
[[Benchmarks]]
Name = "ethereum_core"
Repo = "github.com/ethereum/go-ethereum/core"
- Benchmarks = "BenchmarkInsertChain_ring1000_memdb"
+ Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])"
NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
@@ -66,7 +58,6 @@
[[Benchmarks]]
Name = "ethereum_bitutil"
Repo = "github.com/ethereum/go-ethereum/common/bitutil"
- # RunWrapper = ["bar"]
Benchmarks = "Benchmark(BaseTest2KB|FastTest2KB|Encoding4KBVerySparse)"
[[Benchmarks]]
@@ -137,18 +128,18 @@
[[Benchmarks]]
Name = "hugo_helpers"
Repo = "github.com/gohugoio/hugo/helpers"
- Benchmarks = "Benchmark(StripHTML|ReaderContains)"
+ Benchmarks = "Benchmark(StripHTML|TestTruncateWordsToWholeSentence$|EmojiKyokomiSprint)"
GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
- Name = "k8s_api"
- Repo = "k8s.io/kubernetes/pkg/api/testing"
- Benchmarks = "BenchmarkEncodeCodecFromInternalProtobuf"
+ Name = "k8s_cache"
+ Repo = "k8s.io/client-go/tools/cache"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
- Name = "k8s_schedulercache"
- Repo = "k8s.io/kubernetes/pkg/scheduler/internal/cache"
- Benchmarks = "BenchmarkList1kNodes30kPods"
+ Name = "k8s_workqueue"
+ Repo = "k8s.io/client-go/util/workqueue"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "uuid"
@@ -188,14 +179,15 @@
[[Benchmarks]]
Name = "uber_zap"
- # Repo = "github.com/uber-go/zap/benchmarks"
- Repo = "go.uber.org/zap/benchmarks"
- Benchmarks = "BenchmarkAddingFields/(Zap.Sugar|^[ais])"
+ Repo = "go.uber.org/zap/zapcore"
+ Benchmarks = "Benchmark"
+ Version = "@v1.16.1-0.20210329175301-c23abee72d19"
[[Benchmarks]]
Name = "bindata"
Repo = "github.com/kevinburke/go-bindata"
Benchmarks = "Benchmark"
+ Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible"
[[Benchmarks]]
Name = "cespare_mph"
diff --git a/cmd/bent/configs/benchmarks-all.toml b/cmd/bent/configs/benchmarks-all.toml
index 5c6be49..1711551 100644
--- a/cmd/bent/configs/benchmarks-all.toml
+++ b/cmd/bent/configs/benchmarks-all.toml
@@ -1,8 +1,6 @@
# Benchmarks that build, and that run quickly enough.
# The noisy ones are listed at the bottom
-# etcd bleve
-
[[Benchmarks]]
Name = "dustin_broadcast"
Repo = "github.com/dustin/go-broadcast"
@@ -69,12 +67,13 @@
[[Benchmarks]]
Name = "ethereum_core"
Repo = "github.com/ethereum/go-ethereum/core"
- Benchmarks = "Benchmark([IPF]|ChainW)" # ChainRead crashes as of 1.10 on 2018-03-16
+ Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])"
[[Benchmarks]]
Name = "ethereum_sha3"
Repo = "github.com/ethereum/go-ethereum/crypto/sha3"
Benchmarks = "Benchmark"
+ Disabled = true # won't even get
[[Benchmarks]]
Name = "ethereum_ecies"
@@ -99,11 +98,13 @@
Repo = "github.com/ethereum/go-ethereum/whisper/whisperv5"
Benchmarks = "Benchmark"
NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Disabloed = true # won't even get
[[Benchmarks]]
Name = "eolian_dsp"
- Repo = "buddin.us/eolian/dsp"
+ Repo = "github.com/semrekkers/eolian/dsp"
Benchmarks = "Benchmark"
+ Disabled = true # buddin.us is dead, imports need updating
[[Benchmarks]]
Name = "spexs2"
@@ -114,22 +115,25 @@
Name = "minio"
Repo = "github.com/minio/minio/cmd"
Tests = "none" # Don't run these tests; they hang, etc.
- Benchmarks = "BenchmarkGetObject[^V125]" # not Get|Put|List
+ Benchmarks = "Benchmark([^LPGA]|L[^]i|G[^e]|P[^ua]|A[^u]|Parallel([^LPG]|L[^]i|G[^e]|P[^u]))"
[[Benchmarks]]
Name = "gonum_blas_native"
Repo = "gonum.org/v1/gonum/blas/gonum"
Benchmarks = "Benchmark.*Med" # not all benchmarks
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_lapack_native"
Repo = "gonum.org/v1/gonum/lapack/gonum"
- Benchmarks = "BenchmarkDgeev/.*[mt][1234]"
+ Benchmarks = "Benchmark"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_mat"
Repo = "gonum.org/v1/gonum/mat"
Benchmarks = "Benchmark(.*Vec.*10000($|[^0])|.*Mul.*Dense.*1000H)"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "semver"
@@ -144,22 +148,17 @@
[[Benchmarks]]
Name = "hugo_hugolib"
Repo = "github.com/gohugoio/hugo/hugolib"
- Benchmarks = "Benchmark(S[och]|Get|Replace)"
-
-[[Benchmarks]]
- Name = "hugo_hugolib_sitebuilding"
- Repo = "github.com/gohugoio/hugo/hugolib"
- Benchmarks = "BenchmarkSiteBuilding/YAML,num_pages=10,num_tags=10,tags_per_page=20"
-
-[[Benchmarks]]
- Name = "k8s_api"
- Repo = "k8s.io/kubernetes/pkg/api/testing/"
Benchmarks = "Benchmark"
[[Benchmarks]]
- Name = "k8s_schedulercache"
- Repo = "k8s.io/kubernetes/pkg/scheduler/cache"
- Benchmarks = "Benchmark"
+ Name = "k8s_cache"
+ Repo = "k8s.io/client-go/tools/cache"
+ Benchmarks = "B"
+
+[[Benchmarks]]
+ Name = "k8s_workqueue"
+ Repo = "k8s.io/client-go/util/workqueue"
+ Benchmarks = "B"
[[Benchmarks]]
Name = "uuid"
@@ -170,21 +169,25 @@
Name = "gonum_topo"
Repo = "gonum.org/v1/gonum/graph/topo/"
Benchmarks = "Benchmark"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_path"
Repo = "gonum.org/v1/gonum/graph/path/"
Benchmarks = "Benchmark"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_community"
Repo = "gonum.org/v1/gonum/graph/community/"
- Benchmarks = "BenchmarkLouvainDirectedMultiplex"
+ Benchmarks = "Benchmark"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_traverse"
Repo = "gonum.org/v1/gonum/graph/traverse/"
Benchmarks = "Benchmark"
+ BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "capnproto2"
@@ -194,14 +197,15 @@
[[Benchmarks]]
Name = "uber_zap"
- # Repo = "github.com/uber-go/zap/benchmarks"
- Repo = "go.uber.org/zap/benchmarks"
- Benchmarks = "Benchmark"
+ Repo = "go.uber.org/zap/zapcore"
+ Benchmarks = "B"
+ Version = "@v1.16.1-0.20210329175301-c23abee72d19"
[[Benchmarks]]
Name = "bindata"
Repo = "github.com/kevinburke/go-bindata"
Benchmarks = "Benchmark"
+ Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible"
[[Benchmarks]]
Name = "cespare_mph"
@@ -224,10 +228,11 @@
Name = "gtank_ed25519"
Repo = "github.com/gtank/ed25519"
Benchmarks = "Benchmark"
+ Disabled = true # Won't go get properly
[[Benchmarks]]
Name = "nelsam_gxui_interval"
- Repo = "github.com/nelsam/gxui/interval"
+ Repo = "git.sr.ht/~nelsam/gxui/interval"
Benchmarks = "Benchmark"
[[Benchmarks]]
@@ -255,38 +260,3 @@
Repo = "gitlab.com/golang-commonmark/markdown"
Benchmarks = "Benchmark"
-
-# These are the known-noisy benchmarks
-#
-# AddScaledVec10000Inc1-12 4.58µs ±19% 4.55µs ±13% 4.83µs ±12% 4.77µs ±15%
-# AddScaledVec10000Inc2-12 10.7µs ±10% 10.9µs ± 4% 10.4µs ± 8% 10.6µs ±10%
-# AddVec10000Inc1-12 4.56µs ±19% 4.54µs ±12% 4.86µs ±13% 4.78µs ±16%
-# AddVec10000Inc2-12 10.8µs ± 5% 10.9µs ± 4% 10.3µs ± 7% 10.5µs ± 7%
-# AddVec10000Inc2-12 10.8µs ± 5% 10.9µs ± 4% 10.3µs ± 7% 10.5µs ± 7%
-# BaseTest4KB-12 1.21µs ±11% 1.19µs ± 0% 2.40µs ± 0% 3.23µs ± 1%
-# CheckVersionWildcard-12 37.2ns ±11% 35.7ns ± 0% 37.1ns ± 5% 37.6ns ±12%
-# DgemmMedMedMedNTT-12 187µs ± 6% 191µs ±11% 193µs ± 8% 189µs ± 7%
-# GetObject100KbFS-12 274µs ± 3% 262µs ±14% 278µs ± 3% 282µs ± 1%
-# GetObject10KbFS-12 104µs ± 2% 94µs ±24% 105µs ± 3% 106µs ± 2%
-# GetObject10KbXL-12 5.48ms ± 4% 5.43ms ± 4% 5.48ms ± 5% 5.52ms ± 5%
-# GetObjectParallel100KbFS-12 68.1µs ± 3% 63.9µs ±18% 69.1µs ± 2% 69.7µs ± 3%
-# GetObjectParallel10KbFS-12 48.1µs ± 4% 43.5µs ±27% 48.6µs ± 2% 48.6µs ± 2%
-# GetObjectParallel10MbFS-12 931µs ±13% 862µs ±24% 929µs ± 4% 929µs ± 3%
-# GetObjectParallel1MbFS-12 203µs ± 3% 195µs ±11% 205µs ± 2% 204µs ± 2%
-# GetObjectParallel25MbFS-12 2.28ms ±28% 2.09ms ±23% 2.21ms ± 9% 2.23ms ± 6%
-# GetObjectParallel50MbFS-12 4.50ms ±33% 4.60ms ±22% 4.41ms ±12% 4.81ms ±47%
-# GetObjectParallel5MbFS-12 531µs ± 6% 496µs ±20% 528µs ± 3% 526µs ± 6%
-# GetObjectParallelVerySmallFS-12 37.3µs ±39% 34.2µs ±34% 43.0µs ± 3% 41.9µs ±11%
-# GetObjectVerySmallFS-12 70.0µs ± 3% 58.1µs ±44% 70.6µs ± 2% 70.8µs ± 3%
-# PrecompiledIdentity/128-Gas=27-12 17.3ns ± 0% 18.0ns ± 6% 17.4ns ± 1% 18.0ns ± 7%
-# Prove-12 632µs ± 7% 617µs ±10% 617µs ±10% 626µs ± 7%
-# ReplaceDivider-12 1.03µs ±13% 1.03µs ± 9% 1.06µs ±13% 1.07µs ±10%
-# ReplaceShortcodeTokens-12 2.72µs ±12% 2.62µs ± 9% 2.82µs ± 1% 2.58µs ±10%
-# ScaleVec10000Inc1-12 2.98µs ± 3% 3.00µs ± 6% 3.12µs ±12% 2.98µs ± 2%
-# ScaleVec10000Inc2-12 5.62µs ±21% 5.60µs ± 5% 5.98µs ±11% 5.88µs ±13%
-# SubVec10000Inc1-12 4.58µs ±19% 4.54µs ±11% 4.84µs ±16% 4.76µs ±15%
-# SubVec10000Inc2-12 10.8µs ±10% 10.9µs ± 5% 10.3µs ± 8% 10.5µs ± 7%
-# ValidateVersionWildcard-12 42.8ns ± 9% 41.3ns ± 6% 43.4ns ± 7% 43.1ns ± 6%
-# VerifyProof-12 12.2µs ± 7% 12.4µs ± 6% 12.4µs ± 4% 12.3µs ± 5%
-
-
diff --git a/cmd/bent/configuration.go b/cmd/bent/configuration.go
index 301c5ec..9bfd68d 100644
--- a/cmd/bent/configuration.go
+++ b/cmd/bent/configuration.go
@@ -186,7 +186,7 @@
cmd.Args = append(cmd.Args, "-gcflags="+config.GcFlags)
}
cmd.Args = append(cmd.Args, bench.Repo)
- cmd.Dir = dirs.build // use module-mode
+ cmd.Dir = bench.BuildDir // use module-mode
cmd.Env = defaultEnv
if !bench.NotSandboxed {
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
