cmd/bent: tweak cronjob.sh towards future benchmarking use
added additional variables to upload
added reference to explicit localizing script
added a few benchmarks; adjusted selection on some
updated baseline version to 1.17
added sample "localfix" scripts for several existing builders
added BENTARCH knob to configurations-cronjob.toml
Change-Id: I8af5745ff211e111d822c725a8af293830a4ab7b
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/357012
Trust: David Chase <drchase@google.com>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go
index 4f77ff9..682e06a 100644
--- a/cmd/bent/bent.go
+++ b/cmd/bent/bent.go
@@ -285,7 +285,7 @@
b := &todo.Benchmarks[i]
s := suites[b.Name]
if s == nil {
- fmt.Printf("Benchmark %sw appearing in %s is not listed in %s\n", b.Name, benchFile, suiteFile)
+ fmt.Printf("Benchmark %s appearing in %s is not listed in %s\n", b.Name, benchFile, suiteFile)
os.Exit(1)
}
update(&b.Repo, s.Repo)
@@ -484,7 +484,7 @@
defaultEnv = inheritEnv(defaultEnv, "HOME")
defaultEnv = inheritEnv(defaultEnv, "SHELL")
for _, e := range os.Environ() {
- if strings.HasPrefix(e, "GO") {
+ if strings.HasPrefix(e, "GO") || strings.HasPrefix(e, "BENT") {
defaultEnv = append(defaultEnv, e)
}
}
diff --git a/cmd/bent/configs/benchmarks-50.toml b/cmd/bent/configs/benchmarks-50.toml
index 427dc56..a00f63b 100644
--- a/cmd/bent/configs/benchmarks-50.toml
+++ b/cmd/bent/configs/benchmarks-50.toml
@@ -18,11 +18,11 @@
[[Benchmarks]]
Name = "ethereum_core"
- Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n])"
+ Benchmarks = "Benchmark([^ICBH]|H[^a]|B[^l]|C[^h]|I[^n]).*10000"
[[Benchmarks]]
Name = "shopify_sarama"
- Benchmarks = "Benchmark"
+ Benchmarks = "BenchmarkBroker" # BenchmarkStick.* are noisy
[[Benchmarks]]
Name = "aws_restjson"
@@ -70,7 +70,7 @@
[[Benchmarks]]
Name = "ethereum_trie"
- Benchmarks = "Benchmark[HCKGU]" # Prove and VerifyProof are noisy
+ Benchmarks = "Benchmark[HCKGU].*/10K" # Prove and VerifyProof are noisy
[[Benchmarks]]
Name = "spexs2"
@@ -103,7 +103,7 @@
[[Benchmarks]]
Name = "k8s_workqueue"
- Benchmarks = "Benchmark"
+ Benchmarks = "BenchmarkParallelizeUntil" # DelayingQueue is noisy
[[Benchmarks]]
Name = "uuid"
@@ -134,6 +134,10 @@
Benchmarks = "Benchmark"
[[Benchmarks]]
+ Name = "uber_tally"
+ Benchmarks = "BucketsEqual|ScopeTaggedNoCachedSubscopes|HistogramAllocation"
+
+[[Benchmarks]]
Name = "bindata"
Benchmarks = "Benchmark"
@@ -179,8 +183,8 @@
Benchmarks = "Benchmark(FtoaRegexTrailing|ParseBigBytes)"
[[Benchmarks]]
- Name = "rcrowley_metrics"
- Benchmarks = "(BenchmarkCompute1000000)" # BenchmarkMetrics is very noisy
+ Name = "bloom_bloom"
+ Benchmarks = "Benchmark[SC]" # avoid "Estimated"
[[Benchmarks]]
Name = "kanzi"
diff --git a/cmd/bent/configs/benchmarks-all.toml b/cmd/bent/configs/benchmarks-all.toml
index 849de21..26c2354 100644
--- a/cmd/bent/configs/benchmarks-all.toml
+++ b/cmd/bent/configs/benchmarks-all.toml
@@ -115,11 +115,11 @@
[[Benchmarks]]
Name = "k8s_cache"
- Benchmarks = "B"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "k8s_workqueue"
- Benchmarks = "B"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "uuid"
@@ -144,11 +144,22 @@
[[Benchmarks]]
Name = "capnproto2"
Benchmarks = "Benchmark"
- GcEnv = ["GO111MODULE=on"]
[[Benchmarks]]
Name = "uber_zap"
- Benchmarks = "B"
+ Benchmarks = "Benchmark"
+
+[[Benchmarks]]
+ Name = "uber_tally"
+ Benchmarks = "Benchmark"
+
+[[Benchmarks]]
+ Name = "bloom_bits"
+ Benchmarks = "Benchmark"
+
+[[Benchmarks]]
+ Name = "bloom_bloom"
+ Benchmarks = "Benchmark"
[[Benchmarks]]
Name = "bindata"
diff --git a/cmd/bent/configs/configurations-cronjob.toml b/cmd/bent/configs/configurations-cronjob.toml
index 74c3043..eac54d0 100644
--- a/cmd/bent/configs/configurations-cronjob.toml
+++ b/cmd/bent/configs/configurations-cronjob.toml
@@ -2,29 +2,37 @@
Name = "Base"
Root = "$ROOT/${BASE}"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
[[Configurations]]
Name = "Tip"
Root = "$ROOT/go-tip/"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
[[Configurations]]
Name = "Base-prof"
Root = "$ROOT/${BASE}"
RunWrapper = ["cpuprofile"]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
Name = "Tip-prof"
Root = "$ROOT/go-tip/"
RunWrapper = ["cpuprofile"]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
Name = "Tip-prof-nopreempt"
Root = "$ROOT/go-tip/"
RunWrapper = ["cpuprofile"]
- RunEnv = ["GODEBUG=asyncpreemptoff=1"]
+ RunEnv = ["GODEBUG=asyncpreemptoff=1", "GOARCH=${BENTARCH}"]
+ GcEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
@@ -32,6 +40,8 @@
GcFlags = "-N -l"
Root = "$ROOT/${BASE}"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
@@ -39,6 +49,8 @@
GcFlags = "-N -l"
Root = "$ROOT/go-tip/"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
@@ -46,6 +58,8 @@
GcFlags = "-l"
Root = "$ROOT/${BASE}"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
[[Configurations]]
@@ -53,4 +67,6 @@
GcFlags = "-l"
Root = "$ROOT/go-tip/"
AfterBuild = [ "benchsize", "benchdwarf" ]
+ GcEnv = ["GOARCH=${BENTARCH}"]
+ RunEnv = ["GOARCH=${BENTARCH}"]
Disabled = true
diff --git a/cmd/bent/configs/suites.toml b/cmd/bent/configs/suites.toml
index cee62be..cff60e8 100644
--- a/cmd/bent/configs/suites.toml
+++ b/cmd/bent/configs/suites.toml
@@ -202,7 +202,22 @@
[[Suites]]
Name = "uber_zap"
Repo = "go.uber.org/zap/zapcore"
- Version = "@v1.16.1-0.20210329175301-c23abee72d19"
+ Version = "@v1.19.1"
+
+[[Suites]]
+ Name = "uber_tally"
+ Repo = "github.com/uber-go/tally"
+ Version = "@v3.4.2+incompatible"
+
+[[Suites]]
+ Name = "bloom_bits"
+ Repo = "github.com/bits-and-blooms/bitset"
+ Version = "@v1.2.1"
+
+[[Suites]]
+ Name = "bloom_bloom"
+ Repo = "github.com/bits-and-blooms/bloom/v3"
+ Version = "@v3.1.0"
[[Suites]]
Name = "bindata"
@@ -261,4 +276,3 @@
Name = "commonmark_markdown"
Repo = "gitlab.com/golang-commonmark/markdown"
Version = "@v0.0.0-20191127184510-91b5b3c99c19"
-
diff --git a/cmd/bent/scripts/cronjob.sh b/cmd/bent/scripts/cronjob.sh
index 0cf275f..275ed63 100755
--- a/cmd/bent/scripts/cronjob.sh
+++ b/cmd/bent/scripts/cronjob.sh
@@ -7,13 +7,15 @@
ROOT="${HOME}/work/bent-cron"
export ROOT
+cd "${ROOT}"
# BASE is the baseline, defined here, assumed checked out and built.
-BASE=Go1.14
+BASE=Go1.17
export BASE
# N is number of benchmarks, B is number of builds
-# Can override these with -N= and -a= on command line.
+# Can override these with -N= and -a= on command line, or in localfix.
+#
N=25
B=25
NNl=0
@@ -21,12 +23,15 @@
Nl=0
Bl=1
+# Adjust N, B, define NUMACTL, set GOMAXPROCS, as necessary.
+if [ -e ./localfix ] ; then
+ . ./localfix
+fi
+
if [ "x${SUITE}" = "x" ] ; then
SUITE="bent-cron"
fi
-cd "${ROOT}"
-
if [ ! -e "${BASE}" ] ; then
echo Missing expected baseline directory "${BASE}" in "${ROOT}", attempting to checkout and build.
base=`echo $BASE | tr G g`
@@ -60,6 +65,7 @@
exit 1
fi
tip=`git log -n 1 --format='%h'`
+tiptime=`git log -n 1 --format='%cI'`
# Get revision for base so there is no ambiguity
cd "${ROOT}"/${BASE}
@@ -70,16 +76,44 @@
cd "${ROOT}"
# For arm64 big.little, might need to prefix with something like:
# GOMAXPROCS=4 numactl -C 2-5 -- ...
-${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@"
+GOARCH="${BENTARCH}" ${NUMACTL} ${PERFLOCK} bent -v -N=${N} -a=${B} -L=bentjobs.log -C=configurations-cronjob.toml -c Base,Tip "$@"
RUN=`tail -1 bentjobs.log | awk -c '{print $1}'`
+runstamp="$RUN"
+bentstamp="$RUN"
+
+# variables for better benchmarking
+denominator_branch="${BASE}"
+denominator_hash="${base}"
+
+numerator_branch="master"
+numerator_hash="$tip"
+numerator_stamp="$tiptime"
+
+builder_id=`uname -n`
+builder_type="${BUILDER_TYPE}"
cd bench
STAMP="stamp-$$"
export STAMP
+
+append () {
+ c=`eval echo $\`echo $1\``
+ echo "$1: $c" >> ${STAMP}
+}
+
echo "suite: ${SUITE}" >> ${STAMP}
-echo "bentstamp: ${RUN}" >> "${STAMP}"
-echo "tip: ${tip}" >> "${STAMP}"
-echo "base: ${base}" >> "${STAMP}"
+append bentstamp
+append tip
+append base
+# new stuff for better benchmarking
+append numerator_branch
+append numerator_hash
+append numerator_stamp
+append denominator_branch
+append denominator_hash
+append builder_id
+append builder_type
+append runstamp
SFX="${RUN}"
@@ -106,16 +140,29 @@
# Debugging build
cd "${ROOT}"
-${PERFLOCK} bent -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl "$@"
+GOARCH="${BENTARCH}" ${NUMACTL} ${PERFLOCK} bent -v -N=${NNl} -a=${BNl} -L=bentjobsNl.log -C=configurations-cronjob.toml -c BaseNl,TipNl
RUN=`tail -1 bentjobsNl.log | awk -c '{print $1}'`
+runstamp="$RUN"
+bentstamp="$RUN"
cd bench
STAMP="stamp-$$"
export STAMP
+
echo "suite: ${SUITE}-Nl" >> ${STAMP}
-echo "bentstamp: ${RUN}" >> "${STAMP}"
-echo "tip: ${tip}" >> "${STAMP}"
-echo "base: ${base}" >> "${STAMP}"
+
+append bentstamp
+append tip
+append base
+# new stuff for better benchmarking
+append numerator_branch
+append numerator_hash
+append numerator_stamp
+append denominator_branch
+append denominator_hash
+append builder_id
+append builder_type
+append runstamp
SFX="${RUN}"
@@ -134,16 +181,27 @@
# No-inline build
cd "${ROOT}"
-${PERFLOCK} bent -U -v -N=${Nl} -a=${Bl} -L=bentjobsl.log -C=configurations-cronjob.toml -c Basel,Tipl "$@"
+${NUMACTL} ${PERFLOCK} bent -v -N=${Nl} -a=${Bl} -L=bentjobsl.log -C=configurations-cronjob.toml -c Basel,Tipl
RUN=`tail -1 bentjobsl.log | awk -c '{print $1}'`
+runstamp="$RUN"
+bentstamp="$RUN"
cd bench
STAMP="stamp-$$"
export STAMP
echo "suite: ${SUITE}-l" >> ${STAMP}
-echo "bentstamp: ${RUN}" >> "${STAMP}"
-echo "tip: ${tip}" >> "${STAMP}"
-echo "base: ${base}" >> "${STAMP}"
+append bentstamp
+append tip
+append base
+# new stuff for better benchmarking
+append numerator_branch
+append numerator_hash
+append numerator_stamp
+append denominator_branch
+append denominator_hash
+append builder_id
+append builder_type
+append runstamp
SFX="${RUN}"
diff --git a/cmd/bent/scripts/localfix-intelbox-1 b/cmd/bent/scripts/localfix-intelbox-1
new file mode 100644
index 0000000..bf58c53
--- /dev/null
+++ b/cmd/bent/scripts/localfix-intelbox-1
@@ -0,0 +1 @@
+BUILDER_TYPE="Xeon-E5-1650v3@3.5Ghz*12"
diff --git a/cmd/bent/scripts/localfix-odroid-c2 b/cmd/bent/scripts/localfix-odroid-c2
new file mode 100644
index 0000000..38e7f93
--- /dev/null
+++ b/cmd/bent/scripts/localfix-odroid-c2
@@ -0,0 +1,4 @@
+B=3
+N=15
+BUILDER_TYPE="Odroid-C2-arm"
+export BENTARCH="arm" # referenced in configurations-cronjob.toml
diff --git a/cmd/bent/scripts/localfix-odroid-n2 b/cmd/bent/scripts/localfix-odroid-n2
new file mode 100644
index 0000000..cf54aea
--- /dev/null
+++ b/cmd/bent/scripts/localfix-odroid-n2
@@ -0,0 +1,5 @@
+NUMACTL='numactl -C 2-5 --'
+export GOMAXPROCS=4
+B=3
+N=15
+BUILDER_TYPE="Odroid-N2-arm64"
