cmd/bent: refactor common benchmark info into "suites.toml"; add versions Common fields in benchmark*toml were a headache, especially w/ the addition of versions. This was a big improvement, all the common info (build flags, gc env, etc) can appear there. Also added interpretation of "temporary pattern" for configured "RunDir" for tests that expect to run in a writeable directory (and also w/o testdata, which seems to be the case for at least one bench). Also sorted through benchmarks to get most (all?) running in the "all" case. Change-Id: Id80695c11466ec86810107c1a4acefcde991bff9 Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/355749 Trust: David Chase <drchase@google.com> Run-TryBot: David Chase <drchase@google.com> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go index f498b06..4f77ff9 100644 --- a/cmd/bent/bent.go +++ b/cmd/bent/bent.go
@@ -50,9 +50,27 @@ Version string // To pin a benchmark at a version. } +type Suite struct { + Name string // Short name for benchmark/test + Contact string // Contact not used, but may be present in description + Repo string // Repo + subdir where test resides, used for "go get -t -d ..." + Tests string // Tests to run (regex for -test.run= ) + Benchmarks string // Benchmarks to run (regex for -test.bench= ) + GcEnv []string // Environment variables supplied to 'go test -c' for building, getting + BuildFlags []string // Flags for building test (e.g., -tags purego) + RunWrapper []string // (Inner) Command and args to precede whatever the operation is; may fail in the sandbox. + // e.g. benchmark may run as ConfigWrapper ConfigArg BenchWrapper BenchArg ActualBenchmark + NotSandboxed bool // True if this benchmark cannot or should not be run in a container. + Disabled bool // True if this benchmark is temporarily disabled. + RunDir string // Parent directory of testdata; + BuildDir string // Location of go.mod for this benchmark; download here, go test -c here. + Version string // To pin a benchmark at a version. +} + type Todo struct { Benchmarks []Benchmark Configurations []Configuration + Suites []Suite } // The length of the path to the root of the git repo, inclusive. @@ -68,9 +86,9 @@ var verbose counterFlag -var benchFile = "benchmarks-50.toml" // default list of benchmarks -var confFile = "configurations.toml" // default list of configurations -var srcPath = "src/github.com/dr2chase/bent" // Used to find configuration files. +var benchFile = "benchmarks-50.toml" // default list of benchmarks +var confFile = "configurations.toml" // default list of configurations +var suiteFile = "suites.toml" // default list of suites var container = "" var N = 1 var list = false @@ -98,6 +116,7 @@ var copyConfigs = []string{ "benchmarks-all.toml", "benchmarks-50.toml", "benchmarks-gc.toml", "benchmarks-gcplus.toml", "benchmarks-trial.toml", "configurations-sample.toml", "configurations-gollvm.toml", "configurations-cronjob.toml", "configurations-cmpjob.toml", + "suites.toml", } var defaultEnv []string @@ -229,13 +248,60 @@ fmt.Printf("There was an error opening or reading file %s: %v\n", confFile, err) os.Exit(1) } + blobS, err := ioutil.ReadFile(suiteFile) + if err != nil { + fmt.Printf("There was an error opening or reading file %s: %v\n", suiteFile, err) + os.Exit(1) + } blob := append(blobB, blobC...) + blob = append(blob, blobS...) err = toml.Unmarshal(blob, todo) if err != nil { fmt.Printf("There was an error unmarshalling %s: %v\n", string(blob), err) os.Exit(1) } + // Copy defaults for benchmarks from suites. + // (old code had these associated with the "benchmarks" files) + suites := make(map[string]*Suite) + + for i, s := range todo.Suites { + suites[s.Name] = &todo.Suites[i] + } + + update := func(a *string, s string) { + if *a == "" { + *a = s + } + } + + updateFlags := func(a *[]string, s []string) { + if *a == nil { + *a = s + } + } + + for i := range todo.Benchmarks { + b := &todo.Benchmarks[i] + s := suites[b.Name] + if s == nil { + fmt.Printf("Benchmark %sw appearing in %s is not listed in %s\n", b.Name, benchFile, suiteFile) + os.Exit(1) + } + update(&b.Repo, s.Repo) + update(&b.Version, s.Version) + update(&b.Tests, s.Tests) + update(&b.Benchmarks, s.Benchmarks) + update(&b.RunDir, s.RunDir) + + b.Disabled = s.Disabled || b.Disabled + b.NotSandboxed = s.NotSandboxed || b.NotSandboxed + + updateFlags(&b.BuildFlags, s.BuildFlags) + updateFlags(&b.GcEnv, s.GcEnv) + + } + var moreArgs []string if flag.NArg() > 0 { for i, arg := range flag.Args() { @@ -781,10 +847,25 @@ } // Initialize RunDir for benchmarks. - for i, bench := range todo.Benchmarks { + for i := range todo.Benchmarks { + bench := &todo.Benchmarks[i] if bench.Disabled { continue } + if bench.RunDir != "" { // allow specification of e.g. tmp; otherwise, uses readonly source dir in module cache (for testdata). + // TODO should it always be a tempdir and just copy testdata to there? + dir, err := os.MkdirTemp("", bench.RunDir) + if err != nil { + fmt.Printf("Could not create temporary dir w/ pattern %s for benchmark %s, err=%v\n", bench.RunDir, bench.Name, err) + } + if verbose > 0 { + fmt.Printf("mkdir %s\n", dir) + fmt.Printf("# Rundir=%s; will be deleted on exit\n", dir) + } + bench.RunDir = dir + defer os.RemoveAll(dir) + continue + } // Obtain directory containing testdata, if any: // Capture output of "go list -f {{.Dir}} $PKG" @@ -812,7 +893,7 @@ // if sandboxed, strip cwd from prefix of rundir. rundir = rundir[len(dirs.wd):] } - todo.Benchmarks[i].RunDir = rundir + bench.RunDir = rundir } var failures []string @@ -1013,8 +1094,11 @@ // Initialize the directory, copying in default benchmarks and sample configurations, and creating a Dockerfile if shouldInit { if perr == nil { - fmt.Printf("It looks like you've already initialized this directory, remove ./gopath/pkg if you want to reinit.\n") - os.Exit(1) + if !force { + fmt.Printf("It looks like you've already initialized this directory, remove ./gopath/pkg if you want to reinit.\n") + os.Exit(1) + } + fmt.Printf("Directory appears to already be initialized, but -f (force) so copying files anyway.\n") } for _, s := range copyExes { copyAsset(scripts, "scripts", s)
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml index 375e65f..427dc56 100644 --- a/cmd/bent/configs/benchmarks-50.toml +++ b/cmd/bent/configs/benchmarks-50.toml
@@ -4,267 +4,189 @@ [[Benchmarks]] Name = "klauspost" - Repo = "github.com/dr2chase/benchmarks/klauspost" Benchmarks = "Benchmark" [[Benchmarks]] Name = "minio" - Repo = "github.com/minio/minio/cmd" Tests = "none" # Don't run these tests; they hang, etc. Benchmarks = "Benchmark(MarshalMsgdataUsageCacheInfo|DecodehealingTracker|DataUpdateTracker|AppendMsg(ReplicateDecision|ResyncTargetsInfo))" GcEnv = ["GO111MODULE=on"] [[Benchmarks]] Name = "hugo_hugolib" - Repo = "github.com/gohugoio/hugo/hugolib" Benchmarks = "Benchmark(ResourceChainPostProcess|ReplaceShortcodeTokens|MergeByLanguage|GetPage$|(Cascade/langs-1-8))" [[Benchmarks]] Name = "ethereum_core" - Repo = "github.com/ethereum/go-ethereum/core" Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "shopify_sarama" - Repo = "github.com/Shopify/sarama" Benchmarks = "Benchmark" - Disabled = true # Has no actual benchmarks. [[Benchmarks]] Name = "aws_restjson" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson" Benchmarks = "Benchmark" - Disabled = true # Has no actual benchmarks. [[Benchmarks]] Name = "aws_restxml" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml" Benchmarks = "Benchmark" - Disabled = true # Has no actual benchmarks. [[Benchmarks]] Name = "aws_jsonrpc" - Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" Benchmarks = "Benchmark" - Disabled = true # Has no actual benchmarks. + +[[Benchmarks]] + Name = "aws_jsonutil" + Benchmarks = "Benchmark" [[Benchmarks]] Name = "tidwall_tile38" - Repo = "github.com/tidwall/tile38/tests" Benchmarks = "Benchmark" - Disabled = true # Has no actual benchmarks. [[Benchmarks]] Name = "ethereum_bitutil" - Repo = "github.com/ethereum/go-ethereum/common/bitutil" Benchmarks = "Benchmark(BaseTest2KB|FastTest2KB|Encoding4KBVerySparse)" [[Benchmarks]] Name = "ethereum_storage" - Repo = "github.com/ethersphere/swarm/storage" Benchmarks = "BenchmarkJoin_8" # SplitPyramid seems to have a bug - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_ethash" - Repo = "github.com/ethereum/go-ethereum/consensus/ethash" Benchmarks = "BenchmarkHashimotoLight" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_sha3" - Repo = "github.com/ethereum/go-ethereum/crypto/sha3" Benchmarks = "BenchmarkSha3_224_MTU" - Disabled = true [[Benchmarks]] Name = "ethereum_ecies" - Repo = "github.com/ethereum/go-ethereum/crypto/ecies" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS - Disabled = true # Not a test or a benchmarks anymore [[Benchmarks]] Name = "ethereum_corevm" - Repo = "github.com/ethereum/go-ethereum/core/vm" Benchmarks = "BenchmarkOpDiv128" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_trie" - Repo = "github.com/ethereum/go-ethereum/trie" Benchmarks = "Benchmark[HCKGU]" # Prove and VerifyProof are noisy - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "spexs2" - Repo = "github.com/egonelbre/spexs2/_benchmark/" Benchmarks = "BenchmarkRun/10k/1" [[Benchmarks]] Name = "gonum_blas_native" - Repo = "gonum.org/v1/gonum/blas/gonum" - BuildFlags = ["-tags", "safe"] Benchmarks = "Benchmark(DasumMediumUnitaryInc|Dnrm2MediumPosInc)" # not all benchmarks [[Benchmarks]] Name = "gonum_lapack_native" - Repo = "gonum.org/v1/gonum/lapack/gonum" - BuildFlags = ["-tags", "safe"] Benchmarks = "BenchmarkDgeev/Circulant10" [[Benchmarks]] Name = "gonum_mat" - Repo = "gonum.org/v1/gonum/mat" - BuildFlags = ["-tags", "safe"] Benchmarks = "Benchmark(MulWorkspaceDense1000Hundredth|ScaleVec10000Inc20)" [[Benchmarks]] Name = "semver" - Repo = "github.com/Masterminds/semver" Benchmarks = "BenchmarkValidateVersionTildeFail" [[Benchmarks]] Name = "hugo_helpers" - Repo = "github.com/gohugoio/hugo/helpers" Benchmarks = "Benchmark(StripHTML|TestTruncateWordsToWholeSentence$|EmojiKyokomiSprint)" GcEnv = ["GO111MODULE=on"] [[Benchmarks]] Name = "k8s_cache" - Repo = "k8s.io/client-go/tools/cache" Benchmarks = "Benchmark" [[Benchmarks]] Name = "k8s_workqueue" - Repo = "k8s.io/client-go/util/workqueue" Benchmarks = "Benchmark" [[Benchmarks]] Name = "uuid" - Repo = "github.com/satori/go.uuid/" Benchmarks = "Benchmark" - Disabled = true # not a benchmark anymore [[Benchmarks]] Name = "gonum_topo" - Repo = "gonum.org/v1/gonum/graph/topo/" - BuildFlags = ["-tags", "safe"] Benchmarks = "Benchmark(TarjanSCCGnp_1000_half|TarjanSCCGnp_10_tenth)" [[Benchmarks]] Name = "gonum_path" - Repo = "gonum.org/v1/gonum/graph/path/" - BuildFlags = ["-tags", "safe"] Benchmarks = "Benchmark(AStarUndirectedmallWorld_10_2_2_2_Heur|Dominators/nested_if_n256)" [[Benchmarks]] Name = "gonum_community" - Repo = "gonum.org/v1/gonum/graph/community/" - BuildFlags = ["-tags", "safe"] Benchmarks = "BenchmarkLouvainDirectedMultiplex" [[Benchmarks]] Name = "gonum_traverse" - Repo = "gonum.org/v1/gonum/graph/traverse/" - BuildFlags = ["-tags", "safe"] Benchmarks = "BenchmarkWalkAllBreadthFirstGnp_(10|1000)_tenth" # more difference by size than anything else [[Benchmarks]] Name = "capnproto2" - Repo = "zombiezen.com/go/capnproto2/" Benchmarks = "Benchmark(TextMovementBetweenSegments|Growth_MultiSegment)" - GcEnv = ["GO111MODULE=on"] [[Benchmarks]] Name = "uber_zap" - Repo = "go.uber.org/zap/zapcore" Benchmarks = "Benchmark" - Version = "@v1.16.1-0.20210329175301-c23abee72d19" [[Benchmarks]] Name = "bindata" - Repo = "github.com/kevinburke/go-bindata" Benchmarks = "Benchmark" - Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible" [[Benchmarks]] Name = "cespare_mph" - Repo = "github.com/cespare/mph" Benchmarks = "BenchmarkBuild" # Used by InfluxDB and Prometheus [[Benchmarks]] Name = "cespare_xxhash" - Repo = "github.com/cespare/xxhash" - BuildFlags = ["-tags", "purego"] Benchmarks = "BenchmarkHashes/.*,direct,string,n=10MB" [[Benchmarks]] Name = "gtank_blake2s" - Repo = "github.com/gtank/blake2s" Benchmarks = "BenchmarkHash8K" [[Benchmarks]] Name = "gtank_ed25519" - Repo = "github.com/gtank/ed25519" Benchmarks = "Benchmark(IsOnCurve|ScalarMult)" - Disabled = true # ed25519/fe.go:11:2: use of internal package github.com/gtank/ristretto255/internal/radix51 not allowed [[Benchmarks]] Name = "nelsam_gxui_interval" - Repo = "git.sr.ht/~nelsam/gxui/interval" Benchmarks = "Benchmark" [[Benchmarks]] Name = "ajstarks_deck_generate" - Repo = "github.com/ajstarks/deck/generate" Benchmarks = "Benchmark(Polygon|Arc)" [[Benchmarks]] Name = "benhoyt_goawk" - Repo = "github.com/benhoyt/goawk/interp" Benchmarks = "BenchmarkR" [[Benchmarks]] Name = "ericlagergren_decimal" - Repo = "github.com/ericlagergren/decimal/benchmarks" Benchmarks = "BenchmarkPi/foo=.*/prec=100" [[Benchmarks]] - Name = "ericlagergren_decimal_x" - Repo = "github.com/ericlagergren/decimal/benchmarks" - Benchmarks = "BenchmarkPi/foo=ericlagergren_.Go./prec=100" - Disabled = true - -[[Benchmarks]] Name = "dustin_broadcast" - Repo = "github.com/dustin/go-broadcast" Benchmarks = "Benchmark[^B]" #Brodcast is noisy [[Benchmarks]] Name = "dustin_humanize" - Repo = "github.com/dustin/go-humanize" Benchmarks = "Benchmark(FtoaRegexTrailing|ParseBigBytes)" [[Benchmarks]] Name = "rcrowley_metrics" - Repo = "github.com/rcrowley/go-metrics" Benchmarks = "(BenchmarkCompute1000000)" # BenchmarkMetrics is very noisy [[Benchmarks]] - Name = "aws_jsonutil" - Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" - Benchmarks = "Benchmark" - -[[Benchmarks]] Name = "kanzi" - Repo = "github.com/flanglet/kanzi-go/benchmark" Benchmarks = "Benchmark(BWTS|FPAQ|LZ|MTFT)$" [[Benchmarks]] Name = "commonmark_markdown" - Repo = "gitlab.com/golang-commonmark/markdown" Benchmarks = "Benchmark(RenderSpec|RenderSpecBlackFriday2)"
diff --git a/cmd/bent/configs/benchmarks-all.toml b/cmd/bent/configs/benchmarks-all.toml index 1711551..e05283d 100644 --- a/cmd/bent/configs/benchmarks-all.toml +++ b/cmd/bent/configs/benchmarks-all.toml
@@ -3,260 +3,199 @@ [[Benchmarks]] Name = "dustin_broadcast" - Repo = "github.com/dustin/go-broadcast" Benchmarks = "Benchmark" [[Benchmarks]] Name = "dustin_humanize" - Repo = "github.com/dustin/go-humanize" Benchmarks = "Benchmark" [[Benchmarks]] Name = "rcrowley_metrics" - Repo = "github.com/rcrowley/go-metrics" Benchmarks = "Benchmark" [[Benchmarks]] Name = "shopify_sarama" - Repo = "github.com/Shopify/sarama" Benchmarks = "Benchmark" [[Benchmarks]] Name = "aws_restjson" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson" Benchmarks = "Benchmark" + Disabled = true # no benchmarks [[Benchmarks]] Name = "aws_restxml" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml" Benchmarks = "Benchmark" + Disabled = true # no benchmarks [[Benchmarks]] Name = "aws_jsonrpc" - Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" Benchmarks = "Benchmark" + Disabled = true # no benchmarks [[Benchmarks]] Name = "aws_jsonutil" - Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" Benchmarks = "Benchmark" [[Benchmarks]] Name = "kanzi" - Repo = "github.com/flanglet/kanzi-go/benchmark" Benchmarks = "Benchmark" [[Benchmarks]] Name = "ethereum_bitutil" - Repo = "github.com/ethereum/go-ethereum/common/bitutil" - # Benchmarks = "Benchmark" - Benchmarks = "Benchmark.*Test" + Benchmarks = "Benchmark" [[Benchmarks]] Name = "ethereum_storage" - Repo = "github.com/ethersphere/swarm/storage" - Benchmarks = "Benchmark(Join|SplitTree)" # SplitPyramid seems to have a bug - NotSandboxed = true # Won't cross-compile to Linux on MacOS + Benchmarks = "Benchmark" # SplitPyramid seems to have a bug [[Benchmarks]] Name = "ethereum_ethash" - Repo = "github.com/ethereum/go-ethereum/consensus/ethash" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_core" - Repo = "github.com/ethereum/go-ethereum/core" Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])" [[Benchmarks]] Name = "ethereum_sha3" - Repo = "github.com/ethereum/go-ethereum/crypto/sha3" Benchmarks = "Benchmark" - Disabled = true # won't even get [[Benchmarks]] Name = "ethereum_ecies" - Repo = "github.com/ethereum/go-ethereum/crypto/ecies" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_corevm" - Repo = "github.com/ethereum/go-ethereum/core/vm" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] Name = "ethereum_trie" - Repo = "github.com/ethereum/go-ethereum/trie" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS [[Benchmarks]] - Name = "ethereum_whisperv5" - Repo = "github.com/ethereum/go-ethereum/whisper/whisperv5" + Name = "ethereum_whisper" Benchmarks = "Benchmark" - NotSandboxed = true # Won't cross-compile to Linux on MacOS - Disabloed = true # won't even get [[Benchmarks]] Name = "eolian_dsp" - Repo = "github.com/semrekkers/eolian/dsp" Benchmarks = "Benchmark" - Disabled = true # buddin.us is dead, imports need updating [[Benchmarks]] Name = "spexs2" - Repo = "github.com/egonelbre/spexs2/_benchmark/" Benchmarks = "BenchmarkRun" [[Benchmarks]] Name = "minio" - Repo = "github.com/minio/minio/cmd" Tests = "none" # Don't run these tests; they hang, etc. Benchmarks = "Benchmark([^LPGA]|L[^]i|G[^e]|P[^ua]|A[^u]|Parallel([^LPG]|L[^]i|G[^e]|P[^u]))" [[Benchmarks]] Name = "gonum_blas_native" - Repo = "gonum.org/v1/gonum/blas/gonum" - Benchmarks = "Benchmark.*Med" # not all benchmarks - BuildFlags = ["-tags", "safe"] + Benchmarks = "Benchmark" # not all benchmarks [[Benchmarks]] Name = "gonum_lapack_native" +<<<<<<< HEAD Repo = "gonum.org/v1/gonum/lapack/gonum" +======= +>>>>>>> 8dbe4b9 (cmd/bent: refactor common benchmark info into "suites.toml"; add versions) Benchmarks = "Benchmark" - BuildFlags = ["-tags", "safe"] [[Benchmarks]] Name = "gonum_mat" - Repo = "gonum.org/v1/gonum/mat" - Benchmarks = "Benchmark(.*Vec.*10000($|[^0])|.*Mul.*Dense.*1000H)" - BuildFlags = ["-tags", "safe"] + Benchmarks = "Benchmark" [[Benchmarks]] Name = "semver" - Repo = "github.com/Masterminds/semver" Benchmarks = "Benchmark" [[Benchmarks]] Name = "hugo_helpers" - Repo = "github.com/gohugoio/hugo/helpers" Benchmarks = "Benchmark" [[Benchmarks]] Name = "hugo_hugolib" - Repo = "github.com/gohugoio/hugo/hugolib" Benchmarks = "Benchmark" [[Benchmarks]] Name = "k8s_cache" - Repo = "k8s.io/client-go/tools/cache" Benchmarks = "B" [[Benchmarks]] Name = "k8s_workqueue" - Repo = "k8s.io/client-go/util/workqueue" Benchmarks = "B" [[Benchmarks]] Name = "uuid" - Repo = "github.com/satori/go.uuid/" Benchmarks = "Benchmark" [[Benchmarks]] Name = "gonum_topo" - Repo = "gonum.org/v1/gonum/graph/topo/" Benchmarks = "Benchmark" - BuildFlags = ["-tags", "safe"] [[Benchmarks]] Name = "gonum_path" - Repo = "gonum.org/v1/gonum/graph/path/" Benchmarks = "Benchmark" - BuildFlags = ["-tags", "safe"] [[Benchmarks]] Name = "gonum_community" - Repo = "gonum.org/v1/gonum/graph/community/" Benchmarks = "Benchmark" - BuildFlags = ["-tags", "safe"] [[Benchmarks]] Name = "gonum_traverse" - Repo = "gonum.org/v1/gonum/graph/traverse/" Benchmarks = "Benchmark" - BuildFlags = ["-tags", "safe"] [[Benchmarks]] Name = "capnproto2" - Repo = "zombiezen.com/go/capnproto2/" Benchmarks = "Benchmark" GcEnv = ["GO111MODULE=on"] [[Benchmarks]] Name = "uber_zap" - Repo = "go.uber.org/zap/zapcore" Benchmarks = "B" - Version = "@v1.16.1-0.20210329175301-c23abee72d19" [[Benchmarks]] Name = "bindata" - Repo = "github.com/kevinburke/go-bindata" Benchmarks = "Benchmark" - Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible" [[Benchmarks]] Name = "cespare_mph" - Repo = "github.com/cespare/mph" Benchmarks = "Benchmark" # Used by InfluxDB and Prometheus [[Benchmarks]] Name = "cespare_xxhash" - Repo = "github.com/cespare/xxhash" - BuildFlags = ["-tags", "purego"] Benchmarks = "Benchmark" [[Benchmarks]] Name = "gtank_blake2s" - Repo = "github.com/gtank/blake2s" Benchmarks = "Benchmark" [[Benchmarks]] Name = "gtank_ed25519" - Repo = "github.com/gtank/ed25519" Benchmarks = "Benchmark" - Disabled = true # Won't go get properly [[Benchmarks]] Name = "nelsam_gxui_interval" - Repo = "git.sr.ht/~nelsam/gxui/interval" Benchmarks = "Benchmark" [[Benchmarks]] Name = "ajstarks_deck_generate" - Repo = "github.com/ajstarks/deck/generate" Benchmarks = "Benchmark" [[Benchmarks]] Name = "benhoyt_goawk" - Repo = "github.com/benhoyt/goawk/interp" Benchmarks = "Benchmark" [[Benchmarks]] Name = "ericlagergren_decimal" - Repo = "github.com/ericlagergren/decimal/benchmarks" - Benchmarks = "Benchmark.*_9" + Benchmarks = "Benchmark" [[Benchmarks]] Name = "tidwall_tile38" - Repo = "github.com/tidwall/tile38/tests" Benchmarks = "Benchmark" [[Benchmarks]] Name = "commonmark_markdown" - Repo = "gitlab.com/golang-commonmark/markdown" Benchmarks = "Benchmark"
diff --git a/cmd/bent/configs/suites.toml b/cmd/bent/configs/suites.toml new file mode 100644 index 0000000..cee62be --- /dev/null +++ b/cmd/bent/configs/suites.toml
@@ -0,0 +1,264 @@ +# A Suite is a short name, a repo, a version, and sometimes some related default values +# for the benchmark specification files. + +[[Suites]] + Name = "dustin_broadcast" + Repo = "github.com/dustin/go-broadcast" + Version = "@v0.0.0-20171205050544-f664265f5a66" + +[[Suites]] + Name = "dustin_humanize" + Repo = "github.com/dustin/go-humanize" + Version = "@v1.0.0" + +[[Suites]] + Name = "rcrowley_metrics" + Repo = "github.com/rcrowley/go-metrics" + Version = "@v0.0.0-20201227073835-cf1acfcdf475" + +[[Suites]] + Name = "shopify_sarama" + Repo = "github.com/Shopify/sarama" + Version = "@v1.30.0" + +[[Suites]] + Name = "aws_restjson" + Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson" + Version = "@v1.41.1" + +[[Suites]] + Name = "aws_restxml" + Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml" + Version = "@v1.41.1" + +[[Suites]] + Name = "aws_jsonrpc" + Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" + Version = "@v1.41.1" + +[[Suites]] + Name = "aws_jsonutil" + Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" + Version = "@v1.41.1" + +[[Suites]] + Name = "kanzi" + Repo = "github.com/flanglet/kanzi-go/benchmark" + Version = "@v1.9.0" + +[[Suites]] + Name = "ethereum_bitutil" + Repo = "github.com/ethereum/go-ethereum/common/bitutil" + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_storage" + Repo = "github.com/ethersphere/swarm/storage" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v0.5.8" + +[[Suites]] + Name = "ethereum_ethash" + Repo = "github.com/ethereum/go-ethereum/consensus/ethash" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_core" + Repo = "github.com/ethereum/go-ethereum/core" + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_sha3" + Repo = "github.com/ethereum/go-ethereum/crypto/sha3" + Disabled = true # won't even get + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_ecies" + Repo = "github.com/ethereum/go-ethereum/crypto/ecies" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_corevm" + Repo = "github.com/ethereum/go-ethereum/core/vm" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_trie" + Repo = "github.com/ethereum/go-ethereum/trie" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v1.10.9" + +[[Suites]] + Name = "ethereum_whisper" + Repo = "github.com/ethereum/go-ethereum/whisper/whisperv6" + Benchmarks = "Benchmark" + NotSandboxed = true # Won't cross-compile to Linux on MacOS + Version = "@v1.9.2" + +[[Suites]] + Name = "eolian_dsp" + Repo = "github.com/semrekkers/eolian/dsp" + Disabled = true # buddin.us is dead, imports need updating + +[[Suites]] + Name = "spexs2" + Repo = "github.com/egonelbre/spexs2/_benchmark/" + Version = "@v0.0.0-20190702105659-492016b6bcd2" + +[[Suites]] + Name = "klauspost" + Repo = "github.com/dr2chase/benchmarks/klauspost" + Version = "@v0.0.0-20211005202936-58b13036cdde" + +[[Suites]] + Name = "minio" + Repo = "github.com/minio/minio/cmd" + Tests = "none" # Don't run these tests; they hang, etc. + Version = "@v0.0.0-20211012165317-13e41f2c6899" + +[[Suites]] + Name = "gonum_blas_native" + Repo = "gonum.org/v1/gonum/blas/gonum" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "gonum_lapack_native" + Repo = "gonum.org/v1/gonum/lapack/gonum" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "gonum_mat" + Repo = "gonum.org/v1/gonum/mat" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "semver" + Repo = "github.com/Masterminds/semver" + Version = "@v1.5.0" + +[[Suites]] + Name = "hugo_helpers" + Repo = "github.com/gohugoio/hugo/helpers" + Version = "@v0.88.1" + +[[Suites]] + Name = "hugo_hugolib" + Repo = "github.com/gohugoio/hugo/hugolib" + Version = "@v0.88.1" + +[[Suites]] + Name = "k8s_cache" + Repo = "k8s.io/client-go/tools/cache" + Version = "@v0.22.2" + +[[Suites]] + Name = "k8s_workqueue" + Repo = "k8s.io/client-go/util/workqueue" + Version = "@v0.22.2" + +[[Suites]] + Name = "uuid" + Repo = "github.com/satori/go.uuid/" + Version = "@v1.2.0" + Disabled = true # not a benchmark anymore + +[[Suites]] + Name = "gonum_topo" + Repo = "gonum.org/v1/gonum/graph/topo/" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "gonum_path" + Repo = "gonum.org/v1/gonum/graph/path/" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "gonum_community" + Repo = "gonum.org/v1/gonum/graph/community/" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "gonum_traverse" + Repo = "gonum.org/v1/gonum/graph/traverse/" + BuildFlags = ["-tags", "safe"] + Version = "@v0.9.3" + +[[Suites]] + Name = "capnproto2" + Repo = "zombiezen.com/go/capnproto2/" + GcEnv = ["GO111MODULE=on"] + Version = "@v2.18.2+incompatible" + +[[Suites]] + Name = "uber_zap" + Repo = "go.uber.org/zap/zapcore" + Version = "@v1.16.1-0.20210329175301-c23abee72d19" + +[[Suites]] + Name = "bindata" + Repo = "github.com/kevinburke/go-bindata" + Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible" + +[[Suites]] + Name = "cespare_mph" + Repo = "github.com/cespare/mph" + Version = "@v0.1.0" + +# Used by InfluxDB and Prometheus +[[Suites]] + Name = "cespare_xxhash" + Repo = "github.com/cespare/xxhash" + BuildFlags = ["-tags", "purego"] + Version = "@v1.1.0" + +[[Suites]] + Name = "gtank_blake2s" + Repo = "github.com/gtank/blake2s" + Version = "@v0.0.0-20180603041446-333f0593be06" + +[[Suites]] + Name = "gtank_ed25519" + Repo = "github.com/gtank/ed25519" + Disabled = true # Won't go get properly + +[[Suites]] + Name = "nelsam_gxui_interval" + Repo = "git.sr.ht/~nelsam/gxui/interval" + Version = "@v0.2.0" + +[[Suites]] + Name = "ajstarks_deck_generate" + Repo = "github.com/ajstarks/deck/generate" + Version = "@v0.0.0-20211012005819-a09e9c2a9a29" + +[[Suites]] + Name = "benhoyt_goawk" + Repo = "github.com/benhoyt/goawk/interp" + Version = "@v1.8.1" + +[[Suites]] + Name = "ericlagergren_decimal" + Repo = "github.com/ericlagergren/decimal/benchmarks" + Version = "@v0.0.0-20210307182354-5f8425a47c58" + +[[Suites]] + Name = "tidwall_tile38" + Repo = "github.com/tidwall/tile38/tests" + Version = "@v0.0.0-20211002002505-72b3683f27da" + RunDir = "tile38" + +[[Suites]] + Name = "commonmark_markdown" + Repo = "gitlab.com/golang-commonmark/markdown" + Version = "@v0.0.0-20191127184510-91b5b3c99c19" +