cmd/bent: refresh versions of all the bent benchmarks These are all generally now the latest versions. Problematic benchmarks are disabled in suites.toml. Removed unnecessary -d flag from go get. Bumped default module build version up to 1.26. benchmarks-50 and benchmarks-all have been vetted; others may be stale. benchmarks-50 currently runs too many (387) benchmarks, however fixing that is a problemn for another CL. Updates #80564. Change-Id: I4bc60da7ca1ce203ac3d87ec4b45b2487b829846 Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/806540 Reviewed-by: Junyang Shao <shaojunyang@google.com> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go index 8d3b1b4..14ec62c 100644 --- a/cmd/bent/bent.go +++ b/cmd/bent/bent.go
@@ -81,8 +81,8 @@ var explicitAll counterFlag // Include "-a" on "go test -c" test build ; repeating flag causes multiple rebuilds, useful for build benchmarking. var shuffle = 2 // Dimensionality of (build) shuffling; 0 = none, 1 = per-benchmark, configuration ordering, 2 = bench, config pairs, 3 = across repetitions. var reportBuildTime = true -var experiment = false // Don't reset go.mod, for testing purposes -var minGoVersion = "1.22" // This is the release the toolchain started caring about versions of Go that are too new. +var experiment = false // Don't reset go.mod, for testing purposes +var minGoVersion = "1.26" //go:embed scripts/* var scripts embed.FS @@ -562,7 +562,7 @@ fmt.Print("Go getting") } - // Obtain (go get -d -t -v bench.Repo) all benchmarks, once, populating src + // Obtain (go get -t -v bench.Repo) all benchmarks, once, populating src for i := range todo.Benchmarks { bench := &todo.Benchmarks[i] @@ -610,7 +610,7 @@ fmt.Print(".") } - cmd := exec.Command("go", "get", "-d", "-t", "-v", bench.Repo+bench.Version) + cmd := exec.Command("go", "get", "-t", "-v", bench.Repo+bench.Version) cmd.Env = DefaultEnv() cmd.Dir = bench.BuildDir
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml index 5bf3220..2cd0779 100644 --- a/cmd/bent/configs/benchmarks-50.toml +++ b/cmd/bent/configs/benchmarks-50.toml
@@ -4,7 +4,7 @@ [[Benchmarks]] Name = "wazero" - Benchmarks = "BenchmarkInvocation/interpreter/(string_manipulation_size_50|fib_for_20|random_mat_mul_size_20)|BenchmarkCompilation/with" + Benchmarks = "BenchmarkInvocation/interpreter/(string_manipulation_size_50|fib_for_20|random_mat_mul_size_20)|BenchmarkCompilation/without" [[Benchmarks]] Name = "toml" @@ -22,7 +22,7 @@ [[Benchmarks]] Name = "ethereum_core" - Benchmarks = "BenchmarkP.*10000" + Benchmarks = "Benchmark(InsertChain_ring1000_memdb|BlockChain_1x1000ValueTransferToExisting)" [[Benchmarks]] Name = "shopify_sarama" @@ -41,16 +41,12 @@ Benchmarks = "Benchmark" [[Benchmarks]] - Name = "aws_jsonutil" - Benchmarks = "Benchmark" - -[[Benchmarks]] Name = "ethereum_bitutil" Benchmarks = "Benchmark(BaseTest2KB|FastTest2KB|Encoding4KBVerySparse)" [[Benchmarks]] Name = "ethereum_ethash" - Benchmarks = "BenchmarkHashimotoLight" + Benchmarks = "BenchmarkDifficultyCalculator/big-homestead-16" [[Benchmarks]] Name = "ethereum_sha3" @@ -90,7 +86,7 @@ [[Benchmarks]] Name = "k8s_cache" - Benchmarks = "Benchmark" + Benchmarks = "Benchmark[^S]" # BenchmarkSharedIndexInformer spawns an non-exiting goroutine [[Benchmarks]] Name = "k8s_workqueue" @@ -106,7 +102,11 @@ [[Benchmarks]] Name = "gonum_path" - Benchmarks = "Benchmark(AStarUndirectedmallWorld_10_2_2_2_Heur|Dominators/nested_if_n256)" + Benchmarks = "Benchmark(AStarUndirected|AllShortest)/(NSW_Undirected_100_5_20_2_heuristic|FloydWarshall_AllBetweenFunc_1000×4[|]0.25.1137|DijkstraAllPaths_AllBetween_1000×16[|]0.1.828)" +# BenchmarkAStarUndirected/NSW_Undirected_100_5_20_2_heuristic +# BenchmarkBellmanFordFrom/500_tenth_incremental +# BenchmarkAllShortest/FloydWarshall_AllBetweenFunc_1000×4|0.25(1137) +# BenchmarkAllShortest/DijkstraAllPaths_AllBetween_1000×16|0.1(828) [[Benchmarks]] Name = "gonum_community" @@ -154,7 +154,7 @@ Benchmarks = "Benchmark(Polygon|Arc)" [[Benchmarks]] - Name = "benhoyt_goawk_1_18" + Name = "benhoyt_goawk" Benchmarks = "BenchmarkR" [[Benchmarks]]
diff --git a/cmd/bent/configs/benchmarks-all.toml b/cmd/bent/configs/benchmarks-all.toml index eb924c4..c918765 100644 --- a/cmd/bent/configs/benchmarks-all.toml +++ b/cmd/bent/configs/benchmarks-all.toml
@@ -20,21 +20,14 @@ [[Benchmarks]] Name = "aws_restjson" Benchmarks = "Benchmark" - Disabled = true # no benchmarks [[Benchmarks]] Name = "aws_restxml" Benchmarks = "Benchmark" - Disabled = true # no benchmarks [[Benchmarks]] Name = "aws_jsonrpc" Benchmarks = "Benchmark" - Disabled = true # no benchmarks - -[[Benchmarks]] - Name = "aws_jsonutil" - Benchmarks = "Benchmark" [[Benchmarks]] Name = "kanzi" @@ -50,7 +43,7 @@ [[Benchmarks]] Name = "ethereum_core" - Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])" + Benchmarks = "Benchmark" [[Benchmarks]] Name = "ethereum_sha3" @@ -107,11 +100,11 @@ [[Benchmarks]] Name = "hugo_hugolib" - Benchmarks = "Benchmark" + Benchmarks = "Benchmark([^P]|Page[^P])" # not BenchmarkPagePageCollections [[Benchmarks]] Name = "k8s_cache" - Benchmarks = "Benchmark" + Benchmarks = "Benchmark[^S]" # BenchmarkSharedIndexInformer starts an endless goroutine [[Benchmarks]] Name = "k8s_workqueue" @@ -123,7 +116,22 @@ [[Benchmarks]] Name = "gonum_topo" - Benchmarks = "Benchmark" + Benchmarks = "Benchmark(Tarjan|UndirectedCyclesInGnp_100|UndirectedCyclesInGnp_10_half|BenchmarkSortGnp_10_tenth|SortPath|SortStabilizedGnp_10_tenth|SortStabilizedPath|BronKerbosch|Clique)" + +# BenchmarkSortGnp_10_tenth-16 159045 7126 ns/op +# FAIL: BenchmarkSortGnp_100_tenth +# FAIL: BenchmarkSortGnp_1000_tenth +# FAIL: BenchmarkSortGnp_10_half +# FAIL: BenchmarkSortGnp_100_half +# FAIL: BenchmarkSortGnp_1000_half + +# BenchmarkSortStabilizedGnp_10_tenth-16 135801 8834 ns/op +# FAIL: BenchmarkSortStabilizedGnp_100_tenth +# FAIL: BenchmarkSortStabilizedGnp_1000_tenth +# FAIL: BenchmarkSortStabilizedGnp_10_half +# FAIL: BenchmarkSortStabilizedGnp_100_half +# FAIL: BenchmarkSortStabilizedGnp_1000_half + [[Benchmarks]] Name = "gonum_path" @@ -187,7 +195,7 @@ Benchmarks = "Benchmark" [[Benchmarks]] - Name = "benhoyt_goawk_1_18" + Name = "benhoyt_goawk" Benchmarks = "Benchmark" [[Benchmarks]] @@ -288,10 +296,20 @@ [[Benchmarks]] Name = "uber_tchannel" - Benchmarks = "Benchmark" - Disabled = true # fails to compile + Benchmarks = "Benchmark([^CR]|Call[^s]|Calls[^C]|R[^e])" + # BenchmarkCallsS + Disabled = true # fails to compile on Darwin at least +# go test -vet=off -c -o /blahblahblah/bent-test/testbin/uber_tchannel_Control -a github.com/uber/tchannel-go ) +# There was an error running 'go test', output = # github.com/uber/tchannel-go [github.com/uber/tchannel-go.test] +# ../../gopath/pkg/mod/github.com/uber/tchannel-go@v1.34.6/sockio_darwin.go:28:6: getSendQueueLen redeclared in this block +# ../../gopath/pkg/mod/github.com/uber/tchannel-go@v1.34.6/sockio_bsd.go:28:6: other declaration of getSendQueueLen [[Benchmarks]] Name = "uber_tchannel_benchmark" Benchmarks = "Benchmark" - Disabled = true # fails to compile + Disabled = true # fails to compile on Darwin at least + +[[Benchmarks]] + Name = "ipaddress" + Benchmarks = "Benchmark" +
diff --git a/cmd/bent/configs/suites.toml b/cmd/bent/configs/suites.toml index 2d0ec17..bf70bd4 100644 --- a/cmd/bent/configs/suites.toml +++ b/cmd/bent/configs/suites.toml
@@ -4,12 +4,12 @@ [[Suites]] Name = "wazero" Repo = "github.com/tetratelabs/wazero/internal/integration_test/bench" - Version = "@v1.7.3" + Version = "@v1.12.0" [[Suites]] Name = "toml" Repo = "github.com/BurntSushi/toml" - Version = "@v1.3.2" + Version = "@v1.6.0" ExtraFiles = ["_example"] [[Suites]] @@ -30,42 +30,45 @@ [[Suites]] Name = "dustin_humanize" Repo = "github.com/dustin/go-humanize" - Version = "@v1.0.0" + Version = "@v1.0.1" [[Suites]] Name = "rcrowley_metrics" Repo = "github.com/rcrowley/go-metrics" - Version = "@v0.0.0-20201227073835-cf1acfcdf475" + Version = "@v0.0.0-20250401214520-65e299d6c5c9" [[Suites]] Name = "shopify_sarama" - Repo = "github.com/Shopify/sarama" - Version = "@v1.30.0" + Repo = "github.com/IBM/sarama" + Version = "@v1.60.0" [[Suites]] Name = "aws_restjson" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restjson" - Version = "@v1.41.1" + Repo = "github.com/aws/aws-sdk-go-v2/internal/protocoltest/awsrestjson" + Version = "@release-2026-07-24" + Disabled = true # fails to get; too large for proxy/sumdb server? [[Suites]] Name = "aws_restxml" - Repo = "github.com/aws/aws-sdk-go/private/protocol/restxml" - Version = "@v1.41.1" + Repo = "github.com/aws/aws-sdk-go-v2/internal/protocoltest/restxml" + Version = "@release-2026-07-24" + Disabled = true # fails to get; too large for proxy/sumdb server? [[Suites]] Name = "aws_jsonrpc" - Repo = "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" - Version = "@v1.41.1" + Repo = "github.com/aws/aws-sdk-go-v2/internal/protocoltest/jsonrpc" + Version = "@release-2026-07-24" + Disabled = true # fails to get; too large for proxy/sumdb server? [[Suites]] - Name = "aws_jsonutil" - Repo = "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" - Version = "@v1.41.1" + Name = "aws_query" + Repo = "github.com/aws/aws-sdk-go-v2/internal/protocoltest/query" + Version = "@release-2026-07-24" [[Suites]] Name = "kanzi" - Repo = "github.com/flanglet/kanzi-go/benchmark" - Version = "@v1.9.0" + Repo = "github.com/flanglet/kanzi-go/v2/benchmark" + Version = "@v2.3.0" [[Suites]] Name = "ethereum_bitutil" @@ -83,36 +86,36 @@ Name = "ethereum_ethash" Repo = "github.com/ethereum/go-ethereum/consensus/ethash" NotSandboxed = true # Won't cross-compile to Linux on MacOS - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_core" Repo = "github.com/ethereum/go-ethereum/core" - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_sha3" Repo = "github.com/ethereum/go-ethereum/crypto/sha3" Disabled = true # won't even get - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_ecies" Repo = "github.com/ethereum/go-ethereum/crypto/ecies" NotSandboxed = true # Won't cross-compile to Linux on MacOS - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_corevm" Repo = "github.com/ethereum/go-ethereum/core/vm" NotSandboxed = true # Won't cross-compile to Linux on MacOS - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_trie" Repo = "github.com/ethereum/go-ethereum/trie" NotSandboxed = true # Won't cross-compile to Linux on MacOS - Version = "@v1.10.9" + Version = "@v1.17.4" [[Suites]] Name = "ethereum_whisper" @@ -141,80 +144,79 @@ Name = "minio" Repo = "github.com/minio/minio/cmd" Tests = "none" # Don't run these tests; they hang, etc. - Version = "@v0.0.0-20211012165317-13e41f2c6899" + Version = "@RELEASE.2025-10-15T17-29-55Z" [[Suites]] Name = "gonum_blas_native" Repo = "gonum.org/v1/gonum/blas/gonum" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "gonum_lapack_native" Repo = "gonum.org/v1/gonum/lapack/gonum" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "gonum_mat" Repo = "gonum.org/v1/gonum/mat" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "semver" - Repo = "github.com/Masterminds/semver" - Version = "@v1.5.0" + Repo = "github.com/Masterminds/semver/v3" + Version = "@v3.3.1" [[Suites]] Name = "hugo_helpers" Repo = "github.com/gohugoio/hugo/helpers" - Version = "@v0.88.1" + Version = "@v0.164.0" [[Suites]] Name = "hugo_hugolib" Repo = "github.com/gohugoio/hugo/hugolib" - Version = "@v0.88.1" + Version = "@v0.164.0" [[Suites]] Name = "k8s_cache" Repo = "k8s.io/client-go/tools/cache" - Version = "@v0.22.2" + Version = "@v0.36.2" [[Suites]] Name = "k8s_workqueue" Repo = "k8s.io/client-go/util/workqueue" - Version = "@v0.22.2" + Version = "@v0.36.2" [[Suites]] Name = "uuid" - Repo = "github.com/satori/go.uuid/" + Repo = "github.com/google/uuid/" Version = "@v1.2.0" - Disabled = true # not a benchmark anymore [[Suites]] Name = "gonum_topo" Repo = "gonum.org/v1/gonum/graph/topo/" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "gonum_path" Repo = "gonum.org/v1/gonum/graph/path/" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "gonum_community" Repo = "gonum.org/v1/gonum/graph/community/" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "gonum_traverse" Repo = "gonum.org/v1/gonum/graph/traverse/" BuildFlags = ["-tags", "safe"] - Version = "@v0.9.3" + Version = "@v0.17.0" [[Suites]] Name = "capnproto2" @@ -225,27 +227,28 @@ [[Suites]] Name = "uber_zap" Repo = "go.uber.org/zap/zapcore" - Version = "@v1.19.1" + Version = "@v1.27.0" [[Suites]] Name = "uber_tally" - Repo = "github.com/uber-go/tally" - Version = "@v3.4.2+incompatible" + Repo = "github.com/uber-go/tally/v4" + Version = "@v4.1.11" [[Suites]] Name = "bloom_bits" Repo = "github.com/bits-and-blooms/bitset" - Version = "@v1.2.1" + Version = "@v1.22.0" [[Suites]] Name = "bloom_bloom" Repo = "github.com/bits-and-blooms/bloom/v3" - Version = "@v3.1.0" + Version = "@v3.7.0" [[Suites]] Name = "bindata" Repo = "github.com/kevinburke/go-bindata" Version = "@v3.22.1-0.20211006210656-12dca65da1b8+incompatible" + Disabled = true # obsolete [[Suites]] Name = "cespare_mph" @@ -255,9 +258,9 @@ # Used by InfluxDB and Prometheus [[Suites]] Name = "cespare_xxhash" - Repo = "github.com/cespare/xxhash" + Repo = "github.com/cespare/xxhash/v2" BuildFlags = ["-tags", "purego"] - Version = "@v1.1.0" + Version = "@v2.3.0" [[Suites]] Name = "gtank_blake2s" @@ -272,148 +275,156 @@ [[Suites]] Name = "nelsam_gxui_interval" Repo = "git.sr.ht/~nelsam/gxui/interval" - Version = "@v0.2.0" + Version = "@v0.4.0" [[Suites]] Name = "ajstarks_deck_generate" Repo = "github.com/ajstarks/deck/generate" - Version = "@v0.0.0-20211012005819-a09e9c2a9a29" + Version = "@v0.0.0-20260618203703-cf77e587dfaa" # Turns out having benchmarks w/ overlapping packages makes for noisy results. # Do not use this until we deal with that problem. It is, however, an interesting # collection of code, so don't forget about it. [[Suites]] - Name = "benhoyt_goawk" + Name = "benhoyt_goawk_old" Repo = "github.com/benhoyt/goawk/interp" Version = "@v1.8.1" [[Suites]] - Name = "benhoyt_goawk_1_18" + Name = "benhoyt_goawk" Repo = "github.com/benhoyt/goawk/interp" - Version = "@v1.18.0" + Version = "@v1.29.0" [[Suites]] Name = "ericlagergren_decimal" - Repo = "github.com/ericlagergren/decimal/benchmarks" - Version = "@v0.0.0-20210307182354-5f8425a47c58" + Repo = "github.com/ericlagergren/decimal_benchmarks" + Version = "@v0.0.0-20221016195653-12f81834bb00" [[Suites]] Name = "tidwall_tile38" Repo = "github.com/tidwall/tile38/tests" - Version = "@v0.0.0-20211002002505-72b3683f27da" + Version = "@v1.38.0" [[Suites]] Name = "commonmark_markdown" Repo = "gitlab.com/golang-commonmark/markdown" - Version = "@v0.0.0-20191127184510-91b5b3c99c19" + Version = "@v0.0.0-20211110145824-bf3e522c626a" ExtraFiles = ["spec"] [[Suites]] Name = "workiva_augmentedtree" Repo = "github.com/Workiva/go-datastructures/augmentedtree" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_bitarray" Repo = "github.com/Workiva/go-datastructures/bitarray" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_btree_palm" Repo = "github.com/Workiva/go-datastructures/btree/palm" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_btree_plus" Repo = "github.com/Workiva/go-datastructures/btree/plus" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_fibheap" Repo = "github.com/Workiva/go-datastructures/fibheap" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_queue" Repo = "github.com/Workiva/go-datastructures/queue" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_rangetree" Repo = "github.com/Workiva/go-datastructures/rangetree" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_hilbert" Repo = "github.com/Workiva/go-datastructures/rtree/hilbert" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_rangetree_skiplist" Repo = "github.com/Workiva/go-datastructures/rangetree/skiplist" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_set" Repo = "github.com/Workiva/go-datastructures/set" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_slice" Repo = "github.com/Workiva/go-datastructures/slice/skip" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_avl" Repo = "github.com/Workiva/go-datastructures/tree/avl" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_sort" Repo = "github.com/Workiva/go-datastructures/sort" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_ctrie" Repo = "github.com/Workiva/go-datastructures/trie/ctrie" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_dtrie" Repo = "github.com/Workiva/go-datastructures/trie/dtrie" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_triex" Repo = "github.com/Workiva/go-datastructures/trie/xfast" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "workiva_triey" Repo = "github.com/Workiva/go-datastructures/trie/yfast" - Version = "@v1.0.53" + Version = "@v1.1.7" [[Suites]] Name = "thriftrw_gen" Repo = "go.uber.org/thriftrw/gen" - Version = "@v1.29.2" + Version = "@v1.34.0" [[Suites]] Name = "thriftrw_protocol" Repo = "go.uber.org/thriftrw/protocol/binary" - Version = "@v1.29.2" + Version = "@v1.34.0" [[Suites]] Name = "yarpc" Repo = "go.uber.org/yarpc" - Version = "@v1.58.0" + Version = "@v1.89.0" [[Suites]] Name = "uber_tchannel" Repo = "github.com/uber/tchannel-go" - Version = "@v1.22.0" + Version = "@v1.34.6" + Disabled = true # fails to compile [[Suites]] Name = "uber_tchannel_benchmark" Repo = "github.com/uber/tchannel-go/benchmark" - Version = "@v1.22.0" + Version = "@v1.34.6" + Disabled = true # fails to compile + +[[Suites]] + Name = "ipaddress" + Repo = "github.com/seancfoley/ipaddress-go/ipaddr/test" + Version = "@v1.7.1" +