sweet: add end-to-end integration test
This change adds an integration test for Sweet that 1) obtains assets,
2) generates a subset of assets (that don't take an obscenely long
time), 3) runs all the benchmarks exactly once under short mode.
Currently this test takes about 6 minutes to execute. It also uses just
over 4 GiB of memory and about 6 GiB of disk space.
Change-Id: Ia70efd92f0bd70f218b405aff7eb5a9e8ee4c912
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/378276
Reviewed-by: Michael Pratt <mpratt@google.com>
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/go.mod b/go.mod
index e568648..67892fa 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@
require (
cloud.google.com/go/storage v1.18.2
- github.com/BurntSushi/toml v0.3.1
+ github.com/BurntSushi/toml v1.0.0
github.com/biogo/biogo v1.0.4
github.com/biogo/graph v0.0.0-20150317020928-057c1989faed
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f
diff --git a/go.sum b/go.sum
index 6a3614d..b0d9931 100644
--- a/go.sum
+++ b/go.sum
@@ -48,6 +48,8 @@
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
+github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
diff --git a/sweet/cmd/sweet/integration_test.go b/sweet/cmd/sweet/integration_test.go
new file mode 100644
index 0000000..1af2785
--- /dev/null
+++ b/sweet/cmd/sweet/integration_test.go
@@ -0,0 +1,198 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+
+ "golang.org/x/benchmarks/sweet/common"
+ "golang.org/x/sync/semaphore"
+)
+
+func TestSweetEndToEnd(t *testing.T) {
+ if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
+ t.Skip("Sweet is currently only fully supported on linux/amd64")
+ }
+ if testing.Short() {
+ t.Skip("the full Sweet end-to-end experience takes several minutes")
+ }
+ goRoot := os.Getenv("GOROOT")
+ if goRoot == "" {
+ data, err := exec.Command("go", "env", "GOROOT").Output()
+ if err != nil {
+ t.Fatalf("failed to find a GOROOT: %v", err)
+ }
+ goRoot = strings.TrimSpace(string(data))
+ }
+ goTool := &common.Go{
+ Tool: filepath.Join(goRoot, "bin", "go"),
+ Env: common.NewEnvFromEnviron(),
+ }
+
+ // Build sweet.
+ wd, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ sweetRoot := filepath.Dir(filepath.Dir(wd))
+ sweetBin := filepath.Join(sweetRoot, "sweet")
+ if err := goTool.BuildPath(filepath.Join(sweetRoot, "cmd", "sweet"), sweetBin); err != nil {
+ t.Fatal(err)
+ }
+ // We're on a builder, so arrange all this a little differently.
+ // Let's do all our work in the work directory which has a lot
+ // more headroom, and put the compressed assets in /tmp.
+ var tmpDir, assetsCacheDir string
+ if os.Getenv("GO_BUILDER_NAME") != "" {
+ tmpDir = filepath.Join(sweetRoot, "tmp")
+ if err := os.Mkdir(tmpDir, 0777); err != nil {
+ t.Fatal(err)
+ }
+ // Be explicit that we want /tmp, because the builder is
+ // going to try and give us /workdir/tmp which will not
+ // have enough space for us.
+ assetsCacheDir = filepath.Join("/", "tmp", "go-sweet-assets")
+ defer func() {
+ if err := os.RemoveAll(assetsCacheDir); err != nil {
+ t.Errorf("clearing assets cache directory: %v", err)
+ }
+ }()
+ } else {
+ tmpDir, err = os.MkdirTemp("", "go-sweet-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ assetsCacheDir = filepath.Join(tmpDir, "assets")
+ }
+ defer func() {
+ if err := os.RemoveAll(tmpDir); err != nil {
+ t.Errorf("clearing tmp directory: %v", err)
+ }
+ }()
+
+ // Download assets.
+ getCmd := exec.Command(sweetBin, "get",
+ "-auth", "none",
+ "-cache", assetsCacheDir, // Make a full copy so we can mutate it.
+ "-assets-hash-file", filepath.Join(sweetRoot, "assets.hash"),
+ )
+ if output, err := getCmd.CombinedOutput(); err != nil {
+ t.Logf("command output:\n%s", string(output))
+ t.Fatal(err)
+ }
+
+ // TODO(mknyszek): Test regenerating assets. As it stands, the following
+ // parts of the test will fail if the source assets change, since they're
+ // prebuilt and baked into the assets archive. The only recourse is to
+ // first upload the new archive with the prebuilt assets (i.e. run sweet
+ // gen locally), bump the version, and then upload it (i.e. sweet put).
+
+ // Run each benchmark once.
+ benchDir := filepath.Join(sweetRoot, "benchmarks")
+ cfgPath := makeConfigFile(t, goRoot)
+
+ var outputMu sync.Mutex
+ runShard := func(shard, resultsDir, workDir string) {
+ runCmd := exec.Command(sweetBin, "run",
+ "-run", shard,
+ "-shell",
+ "-count", "1",
+ "-cache", assetsCacheDir,
+ "-bench-dir", benchDir,
+ "-results", resultsDir,
+ "-work-dir", workDir,
+ "-short",
+ cfgPath,
+ )
+ output, runErr := runCmd.CombinedOutput()
+
+ outputMu.Lock()
+ defer outputMu.Unlock()
+
+ // Poke at the results directory.
+ matches, err := filepath.Glob(filepath.Join(resultsDir, "*", "go.results"))
+ if err != nil {
+ t.Errorf("failed to search results directory for results: %v", err)
+ }
+ if len(matches) == 0 {
+ t.Log("no results produced.")
+ }
+
+ // Dump additional information in case of error, and
+ // check for reasonable results in the case of no error.
+ for _, match := range matches {
+ benchmark := filepath.Base(filepath.Dir(match))
+ if runErr != nil {
+ t.Logf("output for %s:", benchmark)
+ }
+ data, err := os.ReadFile(match)
+ if err != nil {
+ t.Errorf("failed to read results for %si: %v", benchmark, err)
+ continue
+ }
+ if runErr != nil {
+ t.Log(string(data))
+ continue
+ }
+ // TODO(mknyszek): Check to make sure the results look reasonable.
+ }
+ if runErr != nil {
+ t.Logf("command output:\n%s", string(output))
+ t.Error(runErr)
+ }
+ }
+ // Limit parallelism to conserve memory.
+ sema := semaphore.NewWeighted(2)
+ var wg sync.WaitGroup
+ for i, shard := range []string{
+ "tile38", "go-build", "biogo-igor", "biogo-krishna", "bleve-query",
+ "gvisor", "fogleman-pt", "bleve-index,fogleman-fauxgl,gopher-lua,markdown",
+ } {
+ sema.Acquire(context.Background(), 1)
+ wg.Add(1)
+ go func(i int, shard string) {
+ defer sema.Release(1)
+ defer wg.Done()
+ resultsDir := filepath.Join(tmpDir, fmt.Sprintf("results-%d", i))
+ workDir := filepath.Join(tmpDir, fmt.Sprintf("tmp-%d", i))
+ runShard(shard, resultsDir, workDir)
+ }(i, shard)
+ }
+ wg.Wait()
+}
+
+func makeConfigFile(t *testing.T, goRoot string) string {
+ t.Helper()
+
+ f, err := os.CreateTemp("", "config.toml")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ cfg := common.ConfigFile{
+ Configs: []*common.Config{
+ &common.Config{
+ Name: "go",
+ GoRoot: goRoot,
+ },
+ },
+ }
+ b, err := common.ConfigFileMarshalTOML(&cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.Write(b); err != nil {
+ t.Fatal(err)
+ }
+ return f.Name()
+}
diff --git a/sweet/common/config.go b/sweet/common/config.go
index 63c88c8..3c000a1 100644
--- a/sweet/common/config.go
+++ b/sweet/common/config.go
@@ -5,8 +5,11 @@
package common
import (
+ "bytes"
"fmt"
"path/filepath"
+
+ "github.com/BurntSushi/toml"
)
const ConfigHelp = `
@@ -53,6 +56,40 @@
}
}
+func ConfigFileMarshalTOML(c *ConfigFile) ([]byte, error) {
+ // Unfortunately because the github.com/BurntSushi/toml
+ // package at v1.0.0 doesn't correctly support Marshaler
+ // (see https://github.com/BurntSushi/toml/issues/341)
+ // we can't actually implement Marshaler for ConfigEnv.
+ // So instead we work around this by implementing MarshalTOML
+ // on Config and use dummy types that have a straightforward
+ // mapping that *does* work.
+ type config struct {
+ Name string `toml:"name"`
+ GoRoot string `toml:"goroot"`
+ BuildEnv []string `toml:"envbuild"`
+ ExecEnv []string `toml:"envexec"`
+ }
+ type configFile struct {
+ Configs []*config `toml:"config"`
+ }
+ var cfgs configFile
+ for _, c := range c.Configs {
+ var cfg config
+ cfg.Name = c.Name
+ cfg.GoRoot = c.GoRoot
+ cfg.BuildEnv = c.BuildEnv.Collapse()
+ cfg.ExecEnv = c.ExecEnv.Collapse()
+
+ cfgs.Configs = append(cfgs.Configs, &cfg)
+ }
+ var b bytes.Buffer
+ if err := toml.NewEncoder(&b).Encode(&cfgs); err != nil {
+ return nil, err
+ }
+ return b.Bytes(), nil
+}
+
type ConfigEnv struct {
*Env
}
diff --git a/sweet/common/config_test.go b/sweet/common/config_test.go
new file mode 100644
index 0000000..22b4e06
--- /dev/null
+++ b/sweet/common/config_test.go
@@ -0,0 +1,83 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package common_test
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/BurntSushi/toml"
+ "golang.org/x/benchmarks/sweet/common"
+)
+
+func TestConfigMarshalTOML(t *testing.T) {
+ cfgsBefore := common.ConfigFile{
+ Configs: []*common.Config{
+ &common.Config{
+ Name: "go",
+ GoRoot: "/path/to/my/goroot",
+ // The unmarashaler propagates the environment,
+ // so to make sure this works, let's also seed
+ // from the environment.
+ BuildEnv: common.ConfigEnv{common.NewEnvFromEnviron()},
+ ExecEnv: common.ConfigEnv{common.NewEnvFromEnviron()},
+ },
+ },
+ }
+ b, err := common.ConfigFileMarshalTOML(&cfgsBefore)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var cfgsAfter common.ConfigFile
+ if err := toml.Unmarshal(b, &cfgsAfter); err != nil {
+ t.Fatal(err)
+ }
+ if l := len(cfgsAfter.Configs); l != len(cfgsBefore.Configs) {
+ t.Fatalf("unexpected number of configs: got %d, want %d", l, len(cfgsBefore.Configs))
+ }
+ for i := range cfgsAfter.Configs {
+ cfgBefore := cfgsBefore.Configs[i]
+ cfgAfter := cfgsAfter.Configs[i]
+
+ if cfgBefore.Name != cfgAfter.Name {
+ t.Fatalf("unexpected name: got %s, want %s", cfgAfter.Name, cfgBefore.Name)
+ }
+ if cfgBefore.GoRoot != cfgAfter.GoRoot {
+ t.Fatalf("unexpected GOROOT: got %s, want %s", cfgAfter.GoRoot, cfgBefore.GoRoot)
+ }
+ compareEnvs(t, cfgBefore.BuildEnv.Env, cfgAfter.BuildEnv.Env)
+ compareEnvs(t, cfgBefore.ExecEnv.Env, cfgAfter.ExecEnv.Env)
+ }
+}
+
+func compareEnvs(t *testing.T, a, b *common.Env) {
+ t.Helper()
+
+ aIndex := makeEnvIndex(a)
+ bIndex := makeEnvIndex(b)
+ for aKey, aVal := range aIndex {
+ if bVal, ok := bIndex[aKey]; !ok {
+ t.Errorf("%s in A but not B", aKey)
+ } else if aVal != bVal {
+ t.Errorf("%s has value %s A but %s in B", aKey, aVal, bVal)
+ }
+ }
+ for bKey := range bIndex {
+ if _, ok := aIndex[bKey]; !ok {
+ t.Errorf("%s in B but not A", bKey)
+ }
+ // Don't check values that exist in both. We got that already
+ // in the first pass.
+ }
+}
+
+func makeEnvIndex(a *common.Env) map[string]string {
+ index := make(map[string]string)
+ for _, s := range a.Collapse() {
+ d := strings.IndexRune(s, '=')
+ index[s[:d]] = s[d+1:]
+ }
+ return index
+}
