cmd/bent: refactor common benchmark info into "suites.toml"; add versions
Common fields in benchmark*toml were a headache, especially w/
the addition of versions. This was a big improvement, all the
common info (build flags, gc env, etc) can appear there.
Also added interpretation of "temporary pattern" for configured
"RunDir" for tests that expect to run in a writeable directory
(and also w/o testdata, which seems to be the case for at least
one bench).
Also sorted through benchmarks to get most (all?) running in
the "all" case.
Change-Id: Id80695c11466ec86810107c1a4acefcde991bff9
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/355749
Trust: David Chase <drchase@google.com>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index f498b06..4f77ff9 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -50,9 +50,27 @@
Version string // To pin a benchmark at a version.
}
+type Suite struct {
+ Name string // Short name for benchmark/test
+ Contact string // Contact not used, but may be present in description
+ Repo string // Repo + subdir where test resides, used for "go get -t -d ..."
+ Tests string // Tests to run (regex for -test.run= )
+ Benchmarks string // Benchmarks to run (regex for -test.bench= )
+ GcEnv []string // Environment variables supplied to 'go test -c' for building, getting
+ BuildFlags []string // Flags for building test (e.g., -tags purego)
+ RunWrapper []string // (Inner) Command and args to precede whatever the operation is; may fail in the sandbox.
+ // e.g. benchmark may run as ConfigWrapper ConfigArg BenchWrapper BenchArg ActualBenchmark
+ NotSandboxed bool // True if this benchmark cannot or should not be run in a container.
+ Disabled bool // True if this benchmark is temporarily disabled.
+ RunDir string // Parent directory of testdata;
+ BuildDir string // Location of go.mod for this benchmark; download here, go test -c here.
+ Version string // To pin a benchmark at a version.
+}
+
type Todo struct {
Benchmarks []Benchmark
Configurations []Configuration
+ Suites []Suite
}
// The length of the path to the root of the git repo, inclusive.
@@ -68,9 +86,9 @@
var verbose counterFlag
-var benchFile = "benchmarks-50.toml" // default list of benchmarks
-var confFile = "configurations.toml" // default list of configurations
-var srcPath = "src/github.com/dr2chase/bent" // Used to find configuration files.
+var benchFile = "benchmarks-50.toml" // default list of benchmarks
+var confFile = "configurations.toml" // default list of configurations
+var suiteFile = "suites.toml" // default list of suites
var container = ""
var N = 1
var list = false
@@ -98,6 +116,7 @@
var copyConfigs = []string{
"benchmarks-all.toml", "benchmarks-50.toml", "benchmarks-gc.toml", "benchmarks-gcplus.toml", "benchmarks-trial.toml",
"configurations-sample.toml", "configurations-gollvm.toml", "configurations-cronjob.toml", "configurations-cmpjob.toml",
+ "suites.toml",
}
var defaultEnv []string
@@ -229,13 +248,60 @@
fmt.Printf("There was an error opening or reading file %s: %v\n", confFile, err)
os.Exit(1)
}
+ blobS, err := ioutil.ReadFile(suiteFile)
+ if err != nil {
+ fmt.Printf("There was an error opening or reading file %s: %v\n", suiteFile, err)
+ os.Exit(1)
+ }
blob := append(blobB, blobC...)
+ blob = append(blob, blobS...)
err = toml.Unmarshal(blob, todo)
if err != nil {
fmt.Printf("There was an error unmarshalling %s: %v\n", string(blob), err)
os.Exit(1)
}
+ // Copy defaults for benchmarks from suites.
+ // (old code had these associated with the "benchmarks" files)
+ suites := make(map[string]*Suite)
+
+ for i, s := range todo.Suites {
+ suites[s.Name] = &todo.Suites[i]
+ }
+
+ update := func(a *string, s string) {
+ if *a == "" {
+ *a = s
+ }
+ }
+
+ updateFlags := func(a *[]string, s []string) {
+ if *a == nil {
+ *a = s
+ }
+ }
+
+ for i := range todo.Benchmarks {
+ b := &todo.Benchmarks[i]
+ s := suites[b.Name]
+ if s == nil {
+ fmt.Printf("Benchmark %sw appearing in %s is not listed in %s\n", b.Name, benchFile, suiteFile)
+ os.Exit(1)
+ }
+ update(&b.Repo, s.Repo)
+ update(&b.Version, s.Version)
+ update(&b.Tests, s.Tests)
+ update(&b.Benchmarks, s.Benchmarks)
+ update(&b.RunDir, s.RunDir)
+
+ b.Disabled = s.Disabled || b.Disabled
+ b.NotSandboxed = s.NotSandboxed || b.NotSandboxed
+
+ updateFlags(&b.BuildFlags, s.BuildFlags)
+ updateFlags(&b.GcEnv, s.GcEnv)
+
+ }
+
var moreArgs []string
if flag.NArg() > 0 {
for i, arg := range flag.Args() {
@@ -781,10 +847,25 @@
}
// Initialize RunDir for benchmarks.
- for i, bench := range todo.Benchmarks {
+ for i := range todo.Benchmarks {
+ bench := &todo.Benchmarks[i]
if bench.Disabled {
continue
}
+ if bench.RunDir != "" { // allow specification of e.g. tmp; otherwise, uses readonly source dir in module cache (for testdata).
+ // TODO should it always be a tempdir and just copy testdata to there?
+ dir, err := os.MkdirTemp("", bench.RunDir)
+ if err != nil {
+ fmt.Printf("Could not create temporary dir w/ pattern %s for benchmark %s, err=%v\n", bench.RunDir, bench.Name, err)
+ }
+ if verbose > 0 {
+ fmt.Printf("mkdir %s\n", dir)
+ fmt.Printf("# Rundir=%s; will be deleted on exit\n", dir)
+ }
+ bench.RunDir = dir
+ defer os.RemoveAll(dir)
+ continue
+ }
// Obtain directory containing testdata, if any:
// Capture output of "go list -f {{.Dir}} $PKG"
@@ -812,7 +893,7 @@
// if sandboxed, strip cwd from prefix of rundir.
rundir = rundir[len(dirs.wd):]
}
- todo.Benchmarks[i].RunDir = rundir
+ bench.RunDir = rundir
}
var failures []string
@@ -1013,8 +1094,11 @@
// Initialize the directory, copying in default benchmarks and sample configurations, and creating a Dockerfile
if shouldInit {
if perr == nil {
- fmt.Printf("It looks like you've already initialized this directory, remove ./gopath/pkg if you want to reinit.\n")
- os.Exit(1)
+ if !force {
+ fmt.Printf("It looks like you've already initialized this directory, remove ./gopath/pkg if you want to reinit.\n")
+ os.Exit(1)
+ }
+ fmt.Printf("Directory appears to already be initialized, but -f (force) so copying files anyway.\n")
}
for _, s := range copyExes {
copyAsset(scripts, "scripts", s)
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml
index 375e65f..427dc56 100644
--- a/cmd/bent/configs/benchmarks-50.toml
+++ b/cmd/bent/configs/benchmarks-50.toml
@@ -4,267 +4,189 @@
[[Benchmarks]]
Name = "klauspost"
- Repo = "github.com/dr2chase/benchmarks/klauspost"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "minio"
- Repo = "github.com/minio/minio/cmd"
Tests = "none" # Don't run these tests; they hang, etc.
Benchmarks = "Benchmark(MarshalMsgdataUsageCacheInfo|DecodehealingTracker|DataUpdateTracker|AppendMsg(ReplicateDecision|ResyncTargetsInfo))"
GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
Name = "hugo_hugolib"
- Repo = "github.com/gohugoio/hugo/hugolib"
Benchmarks = "Benchmark(ResourceChainPostProcess|ReplaceShortcodeTokens|MergeByLanguage|GetPage$|(Cascade/langs-1-8))"
[[Benchmarks]]
Name = "ethereum_core"
- Repo = "github.com/ethereum/go-ethereum/core"
Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "shopify_sarama"
- Repo = "github.com/Shopify/sarama"
Benchmarks = "Benchmark"
- Disabled = true # Has no actual benchmarks.
[[Benchmarks]]
Name = "aws_restjson"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson"
Benchmarks = "Benchmark"
- Disabled = true # Has no actual benchmarks.
[[Benchmarks]]
Name = "aws_restxml"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml"
Benchmarks = "Benchmark"
- Disabled = true # Has no actual benchmarks.
[[Benchmarks]]
Name = "aws_jsonrpc"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc"
Benchmarks = "Benchmark"
- Disabled = true # Has no actual benchmarks.
+
+[[Benchmarks]]
+ Name = "aws_jsonutil"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "tidwall_tile38"
- Repo = "github.com/tidwall/tile38/tests"
Benchmarks = "Benchmark"
- Disabled = true # Has no actual benchmarks.
[[Benchmarks]]
Name = "ethereum_bitutil"
- Repo = "github.com/ethereum/go-ethereum/common/bitutil"
Benchmarks = "Benchmark(BaseTest2KB|FastTest2KB|Encoding4KBVerySparse)"
[[Benchmarks]]
Name = "ethereum_storage"
- Repo = "github.com/ethersphere/swarm/storage"
Benchmarks = "BenchmarkJoin_8" # SplitPyramid seems to have a bug
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_ethash"
- Repo = "github.com/ethereum/go-ethereum/consensus/ethash"
Benchmarks = "BenchmarkHashimotoLight"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_sha3"
- Repo = "github.com/ethereum/go-ethereum/crypto/sha3"
Benchmarks = "BenchmarkSha3_224_MTU"
- Disabled = true
[[Benchmarks]]
Name = "ethereum_ecies"
- Repo = "github.com/ethereum/go-ethereum/crypto/ecies"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
- Disabled = true # Not a test or a benchmarks anymore
[[Benchmarks]]
Name = "ethereum_corevm"
- Repo = "github.com/ethereum/go-ethereum/core/vm"
Benchmarks = "BenchmarkOpDiv128"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_trie"
- Repo = "github.com/ethereum/go-ethereum/trie"
Benchmarks = "Benchmark[HCKGU]" # Prove and VerifyProof are noisy
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "spexs2"
- Repo = "github.com/egonelbre/spexs2/_benchmark/"
Benchmarks = "BenchmarkRun/10k/1"
[[Benchmarks]]
Name = "gonum_blas_native"
- Repo = "gonum.org/v1/gonum/blas/gonum"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(DasumMediumUnitaryInc|Dnrm2MediumPosInc)" # not all benchmarks
[[Benchmarks]]
Name = "gonum_lapack_native"
- Repo = "gonum.org/v1/gonum/lapack/gonum"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkDgeev/Circulant10"
[[Benchmarks]]
Name = "gonum_mat"
- Repo = "gonum.org/v1/gonum/mat"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(MulWorkspaceDense1000Hundredth|ScaleVec10000Inc20)"
[[Benchmarks]]
Name = "semver"
- Repo = "github.com/Masterminds/semver"
Benchmarks = "BenchmarkValidateVersionTildeFail"
[[Benchmarks]]
Name = "hugo_helpers"
- Repo = "github.com/gohugoio/hugo/helpers"
Benchmarks = "Benchmark(StripHTML|TestTruncateWordsToWholeSentence$|EmojiKyokomiSprint)"
GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
Name = "k8s_cache"
- Repo = "k8s.io/client-go/tools/cache"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "k8s_workqueue"
- Repo = "k8s.io/client-go/util/workqueue"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "uuid"
- Repo = "github.com/satori/go.uuid/"
Benchmarks = "Benchmark"
- Disabled = true # not a benchmark anymore
[[Benchmarks]]
Name = "gonum_topo"
- Repo = "gonum.org/v1/gonum/graph/topo/"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(TarjanSCCGnp_1000_half|TarjanSCCGnp_10_tenth)"
[[Benchmarks]]
Name = "gonum_path"
- Repo = "gonum.org/v1/gonum/graph/path/"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "Benchmark(AStarUndirectedmallWorld_10_2_2_2_Heur|Dominators/nested_if_n256)"
[[Benchmarks]]
Name = "gonum_community"
- Repo = "gonum.org/v1/gonum/graph/community/"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkLouvainDirectedMultiplex"
[[Benchmarks]]
Name = "gonum_traverse"
- Repo = "gonum.org/v1/gonum/graph/traverse/"
- BuildFlags = ["-tags", "safe"]
Benchmarks = "BenchmarkWalkAllBreadthFirstGnp_(10|1000)_tenth" # more difference by size than anything else
[[Benchmarks]]
Name = "capnproto2"
- Repo = "zombiezen.com/go/capnproto2/"
Benchmarks = "Benchmark(TextMovementBetweenSegments|Growth_MultiSegment)"
- GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
Name = "uber_zap"
- Repo = "go.uber.org/zap/zapcore"
Benchmarks = "Benchmark"
- Version = "@v1.16.1-0.20210329175301-c23abee72d19"
[[Benchmarks]]
Name = "bindata"
- Repo = "github.com/kevinburke/go-bindata"
Benchmarks = "Benchmark"
- Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible"
[[Benchmarks]]
Name = "cespare_mph"
- Repo = "github.com/cespare/mph"
Benchmarks = "BenchmarkBuild"
# Used by InfluxDB and Prometheus
[[Benchmarks]]
Name = "cespare_xxhash"
- Repo = "github.com/cespare/xxhash"
- BuildFlags = ["-tags", "purego"]
Benchmarks = "BenchmarkHashes/.*,direct,string,n=10MB"
[[Benchmarks]]
Name = "gtank_blake2s"
- Repo = "github.com/gtank/blake2s"
Benchmarks = "BenchmarkHash8K"
[[Benchmarks]]
Name = "gtank_ed25519"
- Repo = "github.com/gtank/ed25519"
Benchmarks = "Benchmark(IsOnCurve|ScalarMult)"
- Disabled = true # ed25519/fe.go:11:2: use of internal package github.com/gtank/ristretto255/internal/radix51 not allowed
[[Benchmarks]]
Name = "nelsam_gxui_interval"
- Repo = "git.sr.ht/~nelsam/gxui/interval"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "ajstarks_deck_generate"
- Repo = "github.com/ajstarks/deck/generate"
Benchmarks = "Benchmark(Polygon|Arc)"
[[Benchmarks]]
Name = "benhoyt_goawk"
- Repo = "github.com/benhoyt/goawk/interp"
Benchmarks = "BenchmarkR"
[[Benchmarks]]
Name = "ericlagergren_decimal"
- Repo = "github.com/ericlagergren/decimal/benchmarks"
Benchmarks = "BenchmarkPi/foo=.*/prec=100"
[[Benchmarks]]
- Name = "ericlagergren_decimal_x"
- Repo = "github.com/ericlagergren/decimal/benchmarks"
- Benchmarks = "BenchmarkPi/foo=ericlagergren_.Go./prec=100"
- Disabled = true
-
-[[Benchmarks]]
Name = "dustin_broadcast"
- Repo = "github.com/dustin/go-broadcast"
Benchmarks = "Benchmark[^B]" #Brodcast is noisy
[[Benchmarks]]
Name = "dustin_humanize"
- Repo = "github.com/dustin/go-humanize"
Benchmarks = "Benchmark(FtoaRegexTrailing|ParseBigBytes)"
[[Benchmarks]]
Name = "rcrowley_metrics"
- Repo = "github.com/rcrowley/go-metrics"
Benchmarks = "(BenchmarkCompute1000000)" # BenchmarkMetrics is very noisy
[[Benchmarks]]
- Name = "aws_jsonutil"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
- Benchmarks = "Benchmark"
-
-[[Benchmarks]]
Name = "kanzi"
- Repo = "github.com/flanglet/kanzi-go/benchmark"
Benchmarks = "Benchmark(BWTS|FPAQ|LZ|MTFT)$"
[[Benchmarks]]
Name = "commonmark_markdown"
- Repo = "gitlab.com/golang-commonmark/markdown"
Benchmarks = "Benchmark(RenderSpec|RenderSpecBlackFriday2)"
diff --git a/cmd/bent/configs/benchmarks-all.toml b/cmd/bent/configs/benchmarks-all.toml
index 1711551..e05283d 100644
--- a/cmd/bent/configs/benchmarks-all.toml
+++ b/cmd/bent/configs/benchmarks-all.toml
@@ -3,260 +3,199 @@
[[Benchmarks]]
Name = "dustin_broadcast"
- Repo = "github.com/dustin/go-broadcast"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "dustin_humanize"
- Repo = "github.com/dustin/go-humanize"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "rcrowley_metrics"
- Repo = "github.com/rcrowley/go-metrics"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "shopify_sarama"
- Repo = "github.com/Shopify/sarama"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "aws_restjson"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson"
Benchmarks = "Benchmark"
+ Disabled = true # no benchmarks
[[Benchmarks]]
Name = "aws_restxml"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml"
Benchmarks = "Benchmark"
+ Disabled = true # no benchmarks
[[Benchmarks]]
Name = "aws_jsonrpc"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc"
Benchmarks = "Benchmark"
+ Disabled = true # no benchmarks
[[Benchmarks]]
Name = "aws_jsonutil"
- Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "kanzi"
- Repo = "github.com/flanglet/kanzi-go/benchmark"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "ethereum_bitutil"
- Repo = "github.com/ethereum/go-ethereum/common/bitutil"
- # Benchmarks = "Benchmark"
- Benchmarks = "Benchmark.*Test"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "ethereum_storage"
- Repo = "github.com/ethersphere/swarm/storage"
- Benchmarks = "Benchmark(Join|SplitTree)" # SplitPyramid seems to have a bug
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Benchmarks = "Benchmark" # SplitPyramid seems to have a bug
[[Benchmarks]]
Name = "ethereum_ethash"
- Repo = "github.com/ethereum/go-ethereum/consensus/ethash"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_core"
- Repo = "github.com/ethereum/go-ethereum/core"
Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])"
[[Benchmarks]]
Name = "ethereum_sha3"
- Repo = "github.com/ethereum/go-ethereum/crypto/sha3"
Benchmarks = "Benchmark"
- Disabled = true # won't even get
[[Benchmarks]]
Name = "ethereum_ecies"
- Repo = "github.com/ethereum/go-ethereum/crypto/ecies"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_corevm"
- Repo = "github.com/ethereum/go-ethereum/core/vm"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
Name = "ethereum_trie"
- Repo = "github.com/ethereum/go-ethereum/trie"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
[[Benchmarks]]
- Name = "ethereum_whisperv5"
- Repo = "github.com/ethereum/go-ethereum/whisper/whisperv5"
+ Name = "ethereum_whisper"
Benchmarks = "Benchmark"
- NotSandboxed = true # Won't cross-compile to Linux on MacOS
- Disabloed = true # won't even get
[[Benchmarks]]
Name = "eolian_dsp"
- Repo = "github.com/semrekkers/eolian/dsp"
Benchmarks = "Benchmark"
- Disabled = true # buddin.us is dead, imports need updating
[[Benchmarks]]
Name = "spexs2"
- Repo = "github.com/egonelbre/spexs2/_benchmark/"
Benchmarks = "BenchmarkRun"
[[Benchmarks]]
Name = "minio"
- Repo = "github.com/minio/minio/cmd"
Tests = "none" # Don't run these tests; they hang, etc.
Benchmarks = "Benchmark([^LPGA]|L[^]i|G[^e]|P[^ua]|A[^u]|Parallel([^LPG]|L[^]i|G[^e]|P[^u]))"
[[Benchmarks]]
Name = "gonum_blas_native"
- Repo = "gonum.org/v1/gonum/blas/gonum"
- Benchmarks = "Benchmark.*Med" # not all benchmarks
- BuildFlags = ["-tags", "safe"]
+ Benchmarks = "Benchmark" # not all benchmarks
[[Benchmarks]]
Name = "gonum_lapack_native"
+<<<<<<< HEAD
Repo = "gonum.org/v1/gonum/lapack/gonum"
+=======
+>>>>>>> 8dbe4b9 (cmd/bent: refactor common benchmark info into "suites.toml"; add versions)
Benchmarks = "Benchmark"
- BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_mat"
- Repo = "gonum.org/v1/gonum/mat"
- Benchmarks = "Benchmark(.*Vec.*10000($|[^0])|.*Mul.*Dense.*1000H)"
- BuildFlags = ["-tags", "safe"]
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "semver"
- Repo = "github.com/Masterminds/semver"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "hugo_helpers"
- Repo = "github.com/gohugoio/hugo/helpers"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "hugo_hugolib"
- Repo = "github.com/gohugoio/hugo/hugolib"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "k8s_cache"
- Repo = "k8s.io/client-go/tools/cache"
Benchmarks = "B"
[[Benchmarks]]
Name = "k8s_workqueue"
- Repo = "k8s.io/client-go/util/workqueue"
Benchmarks = "B"
[[Benchmarks]]
Name = "uuid"
- Repo = "github.com/satori/go.uuid/"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "gonum_topo"
- Repo = "gonum.org/v1/gonum/graph/topo/"
Benchmarks = "Benchmark"
- BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_path"
- Repo = "gonum.org/v1/gonum/graph/path/"
Benchmarks = "Benchmark"
- BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_community"
- Repo = "gonum.org/v1/gonum/graph/community/"
Benchmarks = "Benchmark"
- BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "gonum_traverse"
- Repo = "gonum.org/v1/gonum/graph/traverse/"
Benchmarks = "Benchmark"
- BuildFlags = ["-tags", "safe"]
[[Benchmarks]]
Name = "capnproto2"
- Repo = "zombiezen.com/go/capnproto2/"
Benchmarks = "Benchmark"
GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
Name = "uber_zap"
- Repo = "go.uber.org/zap/zapcore"
Benchmarks = "B"
- Version = "@v1.16.1-0.20210329175301-c23abee72d19"
[[Benchmarks]]
Name = "bindata"
- Repo = "github.com/kevinburke/go-bindata"
Benchmarks = "Benchmark"
- Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible"
[[Benchmarks]]
Name = "cespare_mph"
- Repo = "github.com/cespare/mph"
Benchmarks = "Benchmark"
# Used by InfluxDB and Prometheus
[[Benchmarks]]
Name = "cespare_xxhash"
- Repo = "github.com/cespare/xxhash"
- BuildFlags = ["-tags", "purego"]
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "gtank_blake2s"
- Repo = "github.com/gtank/blake2s"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "gtank_ed25519"
- Repo = "github.com/gtank/ed25519"
Benchmarks = "Benchmark"
- Disabled = true # Won't go get properly
[[Benchmarks]]
Name = "nelsam_gxui_interval"
- Repo = "git.sr.ht/~nelsam/gxui/interval"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "ajstarks_deck_generate"
- Repo = "github.com/ajstarks/deck/generate"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "benhoyt_goawk"
- Repo = "github.com/benhoyt/goawk/interp"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "ericlagergren_decimal"
- Repo = "github.com/ericlagergren/decimal/benchmarks"
- Benchmarks = "Benchmark.*_9"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "tidwall_tile38"
- Repo = "github.com/tidwall/tile38/tests"
Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "commonmark_markdown"
- Repo = "gitlab.com/golang-commonmark/markdown"
Benchmarks = "Benchmark"
diff --git a/cmd/bent/configs/suites.toml b/cmd/bent/configs/suites.toml
new file mode 100644
index 0000000..cee62be
--- /dev/null
+++ b/cmd/bent/configs/suites.toml
@@ -0,0 +1,264 @@
+# A Suite is a short name, a repo, a version, and sometimes some related default values
+# for the benchmark specification files.
+
+[[Suites]]
+ Name = "dustin_broadcast"
+ Repo = "github.com/dustin/go-broadcast"
+ Version = "@v0.0.0-20171205050544-f664265f5a66"
+
+[[Suites]]
+ Name = "dustin_humanize"
+ Repo = "github.com/dustin/go-humanize"
+ Version = "@v1.0.0"
+
+[[Suites]]
+ Name = "rcrowley_metrics"
+ Repo = "github.com/rcrowley/go-metrics"
+ Version = "@v0.0.0-20201227073835-cf1acfcdf475"
+
+[[Suites]]
+ Name = "shopify_sarama"
+ Repo = "github.com/Shopify/sarama"
+ Version = "@v1.30.0"
+
+[[Suites]]
+ Name = "aws_restjson"
+ Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson"
+ Version = "@v1.41.1"
+
+[[Suites]]
+ Name = "aws_restxml"
+ Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml"
+ Version = "@v1.41.1"
+
+[[Suites]]
+ Name = "aws_jsonrpc"
+ Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc"
+ Version = "@v1.41.1"
+
+[[Suites]]
+ Name = "aws_jsonutil"
+ Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+ Version = "@v1.41.1"
+
+[[Suites]]
+ Name = "kanzi"
+ Repo = "github.com/flanglet/kanzi-go/benchmark"
+ Version = "@v1.9.0"
+
+[[Suites]]
+ Name = "ethereum_bitutil"
+ Repo = "github.com/ethereum/go-ethereum/common/bitutil"
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_storage"
+ Repo = "github.com/ethersphere/swarm/storage"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v0.5.8"
+
+[[Suites]]
+ Name = "ethereum_ethash"
+ Repo = "github.com/ethereum/go-ethereum/consensus/ethash"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_core"
+ Repo = "github.com/ethereum/go-ethereum/core"
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_sha3"
+ Repo = "github.com/ethereum/go-ethereum/crypto/sha3"
+ Disabled = true # won't even get
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_ecies"
+ Repo = "github.com/ethereum/go-ethereum/crypto/ecies"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_corevm"
+ Repo = "github.com/ethereum/go-ethereum/core/vm"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_trie"
+ Repo = "github.com/ethereum/go-ethereum/trie"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v1.10.9"
+
+[[Suites]]
+ Name = "ethereum_whisper"
+ Repo = "github.com/ethereum/go-ethereum/whisper/whisperv6"
+ Benchmarks = "Benchmark"
+ NotSandboxed = true # Won't cross-compile to Linux on MacOS
+ Version = "@v1.9.2"
+
+[[Suites]]
+ Name = "eolian_dsp"
+ Repo = "github.com/semrekkers/eolian/dsp"
+ Disabled = true # buddin.us is dead, imports need updating
+
+[[Suites]]
+ Name = "spexs2"
+ Repo = "github.com/egonelbre/spexs2/_benchmark/"
+ Version = "@v0.0.0-20190702105659-492016b6bcd2"
+
+[[Suites]]
+ Name = "klauspost"
+ Repo = "github.com/dr2chase/benchmarks/klauspost"
+ Version = "@v0.0.0-20211005202936-58b13036cdde"
+
+[[Suites]]
+ Name = "minio"
+ Repo = "github.com/minio/minio/cmd"
+ Tests = "none" # Don't run these tests; they hang, etc.
+ Version = "@v0.0.0-20211012165317-13e41f2c6899"
+
+[[Suites]]
+ Name = "gonum_blas_native"
+ Repo = "gonum.org/v1/gonum/blas/gonum"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "gonum_lapack_native"
+ Repo = "gonum.org/v1/gonum/lapack/gonum"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "gonum_mat"
+ Repo = "gonum.org/v1/gonum/mat"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "semver"
+ Repo = "github.com/Masterminds/semver"
+ Version = "@v1.5.0"
+
+[[Suites]]
+ Name = "hugo_helpers"
+ Repo = "github.com/gohugoio/hugo/helpers"
+ Version = "@v0.88.1"
+
+[[Suites]]
+ Name = "hugo_hugolib"
+ Repo = "github.com/gohugoio/hugo/hugolib"
+ Version = "@v0.88.1"
+
+[[Suites]]
+ Name = "k8s_cache"
+ Repo = "k8s.io/client-go/tools/cache"
+ Version = "@v0.22.2"
+
+[[Suites]]
+ Name = "k8s_workqueue"
+ Repo = "k8s.io/client-go/util/workqueue"
+ Version = "@v0.22.2"
+
+[[Suites]]
+ Name = "uuid"
+ Repo = "github.com/satori/go.uuid/"
+ Version = "@v1.2.0"
+ Disabled = true # not a benchmark anymore
+
+[[Suites]]
+ Name = "gonum_topo"
+ Repo = "gonum.org/v1/gonum/graph/topo/"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "gonum_path"
+ Repo = "gonum.org/v1/gonum/graph/path/"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "gonum_community"
+ Repo = "gonum.org/v1/gonum/graph/community/"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "gonum_traverse"
+ Repo = "gonum.org/v1/gonum/graph/traverse/"
+ BuildFlags = ["-tags", "safe"]
+ Version = "@v0.9.3"
+
+[[Suites]]
+ Name = "capnproto2"
+ Repo = "zombiezen.com/go/capnproto2/"
+ GcEnv = ["GO111MODULE=on"]
+ Version = "@v2.18.2+incompatible"
+
+[[Suites]]
+ Name = "uber_zap"
+ Repo = "go.uber.org/zap/zapcore"
+ Version = "@v1.16.1-0.20210329175301-c23abee72d19"
+
+[[Suites]]
+ Name = "bindata"
+ Repo = "github.com/kevinburke/go-bindata"
+ Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible"
+
+[[Suites]]
+ Name = "cespare_mph"
+ Repo = "github.com/cespare/mph"
+ Version = "@v0.1.0"
+
+# Used by InfluxDB and Prometheus
+[[Suites]]
+ Name = "cespare_xxhash"
+ Repo = "github.com/cespare/xxhash"
+ BuildFlags = ["-tags", "purego"]
+ Version = "@v1.1.0"
+
+[[Suites]]
+ Name = "gtank_blake2s"
+ Repo = "github.com/gtank/blake2s"
+ Version = "@v0.0.0-20180603041446-333f0593be06"
+
+[[Suites]]
+ Name = "gtank_ed25519"
+ Repo = "github.com/gtank/ed25519"
+ Disabled = true # Won't go get properly
+
+[[Suites]]
+ Name = "nelsam_gxui_interval"
+ Repo = "git.sr.ht/~nelsam/gxui/interval"
+ Version = "@v0.2.0"
+
+[[Suites]]
+ Name = "ajstarks_deck_generate"
+ Repo = "github.com/ajstarks/deck/generate"
+ Version = "@v0.0.0-20211012005819-a09e9c2a9a29"
+
+[[Suites]]
+ Name = "benhoyt_goawk"
+ Repo = "github.com/benhoyt/goawk/interp"
+ Version = "@v1.8.1"
+
+[[Suites]]
+ Name = "ericlagergren_decimal"
+ Repo = "github.com/ericlagergren/decimal/benchmarks"
+ Version = "@v0.0.0-20210307182354-5f8425a47c58"
+
+[[Suites]]
+ Name = "tidwall_tile38"
+ Repo = "github.com/tidwall/tile38/tests"
+ Version = "@v0.0.0-20211002002505-72b3683f27da"
+ RunDir = "tile38"
+
+[[Suites]]
+ Name = "commonmark_markdown"
+ Repo = "gitlab.com/golang-commonmark/markdown"
+ Version = "@v0.0.0-20191127184510-91b5b3c99c19"
+
