sweet: reduce length of TestSweetEndToEnd
This test is failing on the LUCI builders because it's running out the
default timeout, and timeouts for subrepos haven't been ported there
yet.
Surprisingly, this is the only repository that runs out the default
clock. This prompted me to look into where time is spent in this test,
and I noticed that the etcd benchmark didn't use the -short flag at all
(!).
In addition to making use of -short in the etcd benchmark, this change
also fiddles around with other benchmarks' use of the -short flag to
make the test slightly shorter. The net decrease in time spent on my
local machine is 25% (5m -> 3m45s).
The test is also tweaked to allow more of the small benchmarks to run in
parallel during the test.
Finally, this change adds timing logs to the test, so when it next times
out we have a better idea as to why.
Change-Id: I7e394e675fd767d13778d934584c12ce23675c01
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/506556
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
diff --git a/sweet/benchmarks/etcd/main.go b/sweet/benchmarks/etcd/main.go
index e82e176..b132185 100644
--- a/sweet/benchmarks/etcd/main.go
+++ b/sweet/benchmarks/etcd/main.go
@@ -176,6 +176,8 @@
name string
reportName string
args []string
+ longArgs []string // if !config.short
+ shortArgs []string // if config.short
}
var benchmarks = []benchmark{
@@ -190,9 +192,14 @@
"put",
"--key-size=8",
"--sequential-keys",
- "--total=100000",
"--val-size=256",
},
+ longArgs: []string{
+ "--total=100000",
+ },
+ shortArgs: []string{
+ "--total=1000",
+ },
},
{
name: "stm",
@@ -204,9 +211,14 @@
"stm",
"--keys=1000",
"--keys-per-txn=2",
- "--total=100000",
"--val-size=8",
},
+ longArgs: []string{
+ "--total=100000",
+ },
+ shortArgs: []string{
+ "--total=1000",
+ },
},
}
@@ -216,6 +228,11 @@
hosts = append(hosts, inst.host(clientPort))
}
args := append([]string{"--endpoints", strings.Join(hosts, ",")}, cfg.bench.args...)
+ if cfg.short {
+ args = append(args, cfg.bench.shortArgs...)
+ } else {
+ args = append(args, cfg.bench.longArgs...)
+ }
cmd := exec.Command(cfg.benchmarkBin, args...)
var stdout, stderr bytes.Buffer
diff --git a/sweet/benchmarks/gopher-lua/main.go b/sweet/benchmarks/gopher-lua/main.go
index 88c50c7..5e3d735 100644
--- a/sweet/benchmarks/gopher-lua/main.go
+++ b/sweet/benchmarks/gopher-lua/main.go
@@ -64,12 +64,12 @@
if err := s.CallByParam(freq, input, lua.LNumber(1)); err != nil {
return err
}
- if err := s.CallByParam(freq, input, lua.LNumber(2)); err != nil {
- return err
- }
if short {
return nil
}
+ if err := s.CallByParam(freq, input, lua.LNumber(2)); err != nil {
+ return err
+ }
if err := s.CallByParam(count, input, lua.LString("GGT")); err != nil {
return err
}
diff --git a/sweet/benchmarks/tile38/main.go b/sweet/benchmarks/tile38/main.go
index e755fdb..f65870c 100644
--- a/sweet/benchmarks/tile38/main.go
+++ b/sweet/benchmarks/tile38/main.go
@@ -331,7 +331,7 @@
}
iters := 40 * 50000
if cfg.short {
- iters = 1000
+ iters = 100
}
return driver.RunBenchmark(benchName, func(d *driver.B) error {
if driver.DiagnosticEnabled(diagnostics.Trace) {
diff --git a/sweet/cmd/sweet/integration_test.go b/sweet/cmd/sweet/integration_test.go
index d13ed70..a3fb992 100644
--- a/sweet/cmd/sweet/integration_test.go
+++ b/sweet/cmd/sweet/integration_test.go
@@ -14,6 +14,7 @@
"strings"
"sync"
"testing"
+ "time"
"golang.org/x/benchmarks/sweet/common"
"golang.org/x/sync/semaphore"
@@ -26,6 +27,16 @@
if testing.Short() {
t.Skip("the full Sweet end-to-end experience takes several minutes")
}
+
+ // Timing state for timeout debug logging.
+ testStartTime := time.Now()
+ lastTime := testStartTime
+ phaseDone := func(name string) {
+ now := time.Now()
+ t.Logf("phase %s @%s (duration: %s)", name, lastTime.Sub(testStartTime), now.Sub(lastTime))
+ lastTime = now
+ }
+
goRoot := os.Getenv("GOROOT")
if goRoot == "" {
data, err := exec.Command("go", "env", "GOROOT").Output()
@@ -87,6 +98,8 @@
}
}()
+ phaseDone("setup")
+
// Download assets.
getCmd := exec.Command(sweetBin, "get",
"-auth", "none",
@@ -98,6 +111,8 @@
t.Fatal(err)
}
+ phaseDone("sweet-get")
+
// TODO(mknyszek): Test regenerating assets. As it stands, the following
// parts of the test will fail if the source assets change, since they're
// prebuilt and baked into the assets archive. The only recourse is to
@@ -110,6 +125,12 @@
var outputMu sync.Mutex
runShard := func(shard, resultsDir, workDir string) {
+ startTime := time.Now()
+ defer func() {
+ endTime := time.Now()
+ t.Logf("\tphase sweet-run-%s @%s (duration: %s)", shard, startTime.Sub(testStartTime), endTime.Sub(startTime))
+ }()
+
args := []string{
"run",
"-run", shard,
@@ -163,16 +184,26 @@
t.Error(runErr)
}
}
+ type shard struct {
+ run string
+ weight int64
+ }
// Limit parallelism to conserve memory.
- sema := semaphore.NewWeighted(2)
+ sema := semaphore.NewWeighted(8)
var wg sync.WaitGroup
- for i, shard := range []string{
- "tile38", "go-build", "biogo-igor", "biogo-krishna", "etcd",
+ for i, shard := range []shard{
+ {"tile38", 4},
+ {"go-build", 4},
+ {"biogo-igor", 1},
+ {"biogo-krishna", 1},
+ {"etcd", 1},
+ {"bleve-index", 1},
+ {"gopher-lua", 1},
+ {"markdown", 1},
// TODO(go.dev/issue/51445): Enable once gVisor builds with Go 1.19.
- // "gvisor",
- "bleve-index,gopher-lua,markdown",
+ // {"gvisor", 1},
} {
- sema.Acquire(context.Background(), 1)
+ sema.Acquire(context.Background(), shard.weight)
wg.Add(1)
go func(i int, shard string) {
defer sema.Release(1)
@@ -180,9 +211,11 @@
resultsDir := filepath.Join(tmpDir, fmt.Sprintf("results-%d", i))
workDir := filepath.Join(tmpDir, fmt.Sprintf("tmp-%d", i))
runShard(shard, resultsDir, workDir)
- }(i, shard)
+ }(i, shard.run)
}
wg.Wait()
+
+ phaseDone("sweet-run")
}
func makeConfigFile(t *testing.T, goRoot string) string {
diff --git a/sweet/harnesses/local.go b/sweet/harnesses/local.go
index d3a0022..af1d71c 100644
--- a/sweet/harnesses/local.go
+++ b/sweet/harnesses/local.go
@@ -79,14 +79,19 @@
return &localBenchHarness{
binName: "bleve-index-bench",
genArgs: func(cfg *common.Config, rcfg *common.RunConfig) []string {
- args := []string{
- "-batch-size", "100",
- filepath.Join(rcfg.AssetsDir, "enwiki-20080103-pages-articles.xml.bz2"),
- }
+ var args []string
if rcfg.Short {
- args = append([]string{"-documents", "100"}, args...)
+ args = []string{
+ "-documents", "10",
+ "-batch-size", "10",
+ }
+ } else {
+ args = []string{
+ "-documents", "1000",
+ "-batch-size", "100",
+ }
}
- return args
+ return append(args, filepath.Join(rcfg.AssetsDir, "enwiki-20080103-pages-articles.xml.bz2"))
},
}
}
