| // Copyright 2021 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package main_test |
| |
| import ( |
| "context" |
| "fmt" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "runtime" |
| "strings" |
| "sync" |
| "testing" |
| "time" |
| |
| "golang.org/x/benchmarks/sweet/common" |
| "golang.org/x/sync/semaphore" |
| ) |
| |
| func TestSweetEndToEnd(t *testing.T) { |
| if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { |
| t.Skip("Sweet is currently only fully supported on linux/amd64") |
| } |
| if testing.Short() { |
| t.Skip("the full Sweet end-to-end experience takes several minutes") |
| } |
| |
| // Timing state for timeout debug logging. |
| testStartTime := time.Now() |
| lastTime := testStartTime |
| phaseDone := func(name string) { |
| now := time.Now() |
| t.Logf("phase %s @%s (duration: %s)", name, lastTime.Sub(testStartTime), now.Sub(lastTime)) |
| lastTime = now |
| } |
| |
| goRoot := os.Getenv("GOROOT") |
| if goRoot == "" { |
| data, err := exec.Command("go", "env", "GOROOT").Output() |
| if err != nil { |
| t.Fatalf("failed to find a GOROOT: %v", err) |
| } |
| goRoot = strings.TrimSpace(string(data)) |
| } |
| goTool := &common.Go{ |
| Tool: filepath.Join(goRoot, "bin", "go"), |
| Env: common.NewEnvFromEnviron(), |
| } |
| |
| cmd := exec.Command(goTool.Tool, "help", "build") |
| out, err := cmd.Output() |
| if err != nil { |
| t.Fatalf("error running go help build: %v", err) |
| } |
| hasPGO := strings.Contains(string(out), "-pgo") |
| |
| // Build sweet. |
| wd, err := os.Getwd() |
| if err != nil { |
| t.Fatal(err) |
| } |
| sweetRoot := filepath.Dir(filepath.Dir(wd)) |
| sweetBin := filepath.Join(sweetRoot, "sweet") |
| if err := goTool.BuildPath(filepath.Join(sweetRoot, "cmd", "sweet"), sweetBin); err != nil { |
| t.Fatal(err) |
| } |
| // We're on a builder, so arrange all this a little differently. |
| // Let's do all our work in the work directory which has a lot |
| // more headroom, and put the compressed assets in /tmp. |
| var tmpDir, assetsCacheDir string |
| if os.Getenv("GO_BUILDER_NAME") != "" { |
| tmpDir = filepath.Join(sweetRoot, "tmp") |
| if err := os.Mkdir(tmpDir, 0777); err != nil { |
| t.Fatal(err) |
| } |
| // Be explicit that we want /tmp, because the builder is |
| // going to try and give us /workdir/tmp which will not |
| // have enough space for us. |
| assetsCacheDir = filepath.Join("/", "tmp", "go-sweet-assets") |
| defer func() { |
| if err := os.RemoveAll(assetsCacheDir); err != nil { |
| t.Errorf("clearing assets cache directory: %v", err) |
| } |
| }() |
| } else { |
| tmpDir, err = os.MkdirTemp("", "go-sweet-test") |
| if err != nil { |
| t.Fatal(err) |
| } |
| assetsCacheDir = filepath.Join(tmpDir, "assets") |
| } |
| defer func() { |
| if err := os.RemoveAll(tmpDir); err != nil { |
| t.Errorf("clearing tmp directory: %v", err) |
| } |
| }() |
| |
| phaseDone("setup") |
| |
| // Download assets. |
| getCmd := exec.Command(sweetBin, "get", |
| "-auth", "none", |
| "-cache", assetsCacheDir, // Make a full copy so we can mutate it. |
| "-assets-hash-file", filepath.Join(sweetRoot, "assets.hash"), |
| ) |
| if output, err := getCmd.CombinedOutput(); err != nil { |
| t.Logf("command output:\n%s", string(output)) |
| t.Fatal(err) |
| } |
| |
| phaseDone("sweet-get") |
| |
| // TODO(mknyszek): Test regenerating assets. As it stands, the following |
| // parts of the test will fail if the source assets change, since they're |
| // prebuilt and baked into the assets archive. The only recourse is to |
| // first upload the new archive with the prebuilt assets (i.e. run sweet |
| // gen locally), bump the version, and then upload it (i.e. sweet put). |
| |
| // Run each benchmark once. |
| benchDir := filepath.Join(sweetRoot, "benchmarks") |
| cfgPath := makeConfigFile(t, goRoot) |
| |
| var outputMu sync.Mutex |
| runShard := func(shard, resultsDir, workDir string) { |
| startTime := time.Now() |
| defer func() { |
| endTime := time.Now() |
| t.Logf("\tphase sweet-run-%s @%s (duration: %s)", shard, startTime.Sub(testStartTime), endTime.Sub(startTime)) |
| }() |
| |
| args := []string{ |
| "run", |
| "-run", shard, |
| "-shell", |
| "-count", "1", |
| "-cache", assetsCacheDir, |
| "-bench-dir", benchDir, |
| "-results", resultsDir, |
| "-work-dir", workDir, |
| "-short", |
| } |
| if hasPGO { |
| args = append(args, "-pgo", "-pgo-count", "1") |
| } |
| args = append(args, cfgPath) |
| runCmd := exec.Command(sweetBin, args...) |
| output, runErr := runCmd.CombinedOutput() |
| |
| outputMu.Lock() |
| defer outputMu.Unlock() |
| |
| // Poke at the results directory. |
| var resultFiles []string |
| addResultFiles := func(fileName string) { |
| matches, err := filepath.Glob(filepath.Join(resultsDir, "*", fileName)) |
| if err != nil { |
| t.Errorf("failed to search results directory for %s: %v", fileName, err) |
| } else if len(matches) == 0 { |
| t.Logf("no %s results", fileName) |
| } |
| resultFiles = append(resultFiles, matches...) |
| } |
| if hasPGO { |
| addResultFiles("go.profile.results") |
| } |
| addResultFiles("go.results") |
| |
| // Dump additional information in case of error, and |
| // check for reasonable results in the case of no error. |
| for _, resultFile := range resultFiles { |
| benchmark := filepath.Base(filepath.Dir(resultFile)) |
| if runErr != nil { |
| t.Logf("output for %s:", benchmark) |
| logFile := resultFile[:len(resultFile)-len(filepath.Ext(resultFile))] + ".log" |
| log, err := os.ReadFile(logFile) |
| if err != nil { |
| t.Errorf("failed to read log for %s: %v", benchmark, err) |
| continue |
| } |
| t.Log(string(log)) |
| } |
| data, err := os.ReadFile(resultFile) |
| if err != nil { |
| t.Errorf("failed to read results for %s: %v", benchmark, err) |
| continue |
| } |
| // TODO(mknyszek): Do some more exhaustive checking with the benchfmt package. |
| if !strings.Contains(string(data), "Benchmark") { |
| t.Errorf("no benchmark data found in result file for %s", benchmark) |
| } |
| } |
| if runErr != nil { |
| t.Logf("command output:\n%s", string(output)) |
| t.Error(runErr) |
| } |
| } |
| type shard struct { |
| run string |
| weight int64 |
| } |
| // Limit parallelism to conserve memory. |
| sema := semaphore.NewWeighted(8) |
| var wg sync.WaitGroup |
| for i, shard := range []shard{ |
| {"tile38", 2}, |
| {"go-build", 4}, |
| {"cockroachdb", 1}, |
| {"etcd", 1}, |
| {"esbuild", 1}, |
| {"bleve-index", 1}, |
| {"gopher-lua", 1}, |
| {"markdown", 1}, |
| {"gvisor", 1}, |
| } { |
| sema.Acquire(context.Background(), shard.weight) |
| wg.Add(1) |
| go func(i int, shard string) { |
| defer sema.Release(1) |
| defer wg.Done() |
| resultsDir := filepath.Join(tmpDir, fmt.Sprintf("results-%d", i)) |
| workDir := filepath.Join(tmpDir, fmt.Sprintf("tmp-%d", i)) |
| runShard(shard, resultsDir, workDir) |
| }(i, shard.run) |
| } |
| wg.Wait() |
| |
| phaseDone("sweet-run") |
| } |
| |
| func makeConfigFile(t *testing.T, goRoot string) string { |
| t.Helper() |
| |
| f, err := os.CreateTemp("", "config.toml") |
| if err != nil { |
| t.Fatal(err) |
| } |
| defer f.Close() |
| cfg := common.ConfigFile{ |
| Configs: []*common.Config{ |
| { |
| Name: "go", |
| GoRoot: goRoot, |
| }, |
| }, |
| } |
| b, err := common.ConfigFileMarshalTOML(&cfg) |
| if err != nil { |
| t.Fatal(err) |
| } |
| if _, err := f.Write(b); err != nil { |
| t.Fatal(err) |
| } |
| return f.Name() |
| } |