blob: 18f81e4f736544a430c89cb7358be5a9e0d31840 [file] [log] [blame]
Michael Pratt88d4abd2021-10-12 17:44:09 -04001// Copyright 2021 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Binary bench provides a unified wrapper around the different types of
6// benchmarks in x/benchmarks.
Michael Pratt06e21e22022-01-06 18:01:12 -05007//
8// Benchmarks are run against the toolchain in GOROOT, and optionally an
9// additional baseline toolchain in BENCH_BASELINE_GOROOT.
Michael Pratt88d4abd2021-10-12 17:44:09 -040010package main
11
12import (
Michael Pratt4bd3f692022-01-10 12:02:02 -050013 "flag"
Michael Pratt06e21e22022-01-06 18:01:12 -050014 "fmt"
Michael Pratt88d4abd2021-10-12 17:44:09 -040015 "log"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "strings"
Michael Anthony Knyszekb92514a2022-03-16 19:10:16 +000020 "time"
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000021
22 "golang.org/x/benchmarks/sweet/common"
Michael Pratt88d4abd2021-10-12 17:44:09 -040023)
24
Michael Anthony Knyszek4fbb60f2022-02-14 22:49:30 +000025var (
Dung Le6a564b02022-11-30 14:16:16 -050026 wait = flag.Bool("wait", true, "wait for system idle before starting benchmarking")
27 gorootExperiment = flag.String("goroot", "", "GOROOT to test (default $GOROOT or 'go env GOROOT')")
28 gorootBaseline = flag.String("goroot-baseline", "", "baseline GOROOT to test against (optional) (default $BENCH_BASELINE_GOROOT)")
29 branch = flag.String("branch", "", "branch of the commits we're testing against (default $BENCH_BRANCH or unknown)")
30 repository = flag.String("repository", "", "repository name of the commits we're testing against (default $BENCH_REPOSITORY or 'go')")
31 subRepoExperiment = flag.String("subrepo", "", "Sub-repo dir to test (default $BENCH_SUBREPO_PATH)")
32 subRepoBaseline = flag.String("subrepo-baseline", "", "Sub-repo baseline to test against (default $BENCH_SUBREPO_BASELINE_PATH)")
Michael Anthony Knyszek4fbb60f2022-02-14 22:49:30 +000033)
Michael Pratt4bd3f692022-01-10 12:02:02 -050034
Michael Pratt88d4abd2021-10-12 17:44:09 -040035func determineGOROOT() (string, error) {
36 g, ok := os.LookupEnv("GOROOT")
37 if ok {
38 return g, nil
39 }
40
41 cmd := exec.Command("go", "env", "GOROOT")
42 b, err := cmd.Output()
43 if err != nil {
44 return "", err
45 }
46 return strings.TrimSpace(string(b)), nil
47}
48
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000049type toolchain struct {
50 *common.Go
51 Name string
Michael Pratt88d4abd2021-10-12 17:44:09 -040052}
53
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000054func toolchainFromGOROOT(name, goroot string) *toolchain {
55 return &toolchain{
56 Go: &common.Go{
57 Tool: filepath.Join(goroot, "bin", "go"),
58 // Update the GOROOT so the wrong one doesn't propagate from
59 // the environment.
60 Env: common.NewEnvFromEnviron().MustSet("GOROOT=" + goroot),
61 PassOutput: true,
62 },
63 Name: name,
64 }
65}
Michael Pratt88d4abd2021-10-12 17:44:09 -040066
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000067func run(tcs []*toolchain) error {
68 // Because each of the functions below is responsible for running
69 // benchmarks under each toolchain itself, it is also responsible
70 // for ensuring that the benchmark tag "toolchain" is printed.
Michael Pratt88d4abd2021-10-12 17:44:09 -040071 pass := true
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000072 if err := goTest(tcs); err != nil {
Michael Pratt88d4abd2021-10-12 17:44:09 -040073 pass = false
74 log.Printf("Error running Go tests: %v", err)
75 }
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +000076 if err := bent(tcs); err != nil {
Michael Pratt88d4abd2021-10-12 17:44:09 -040077 pass = false
78 log.Printf("Error running bent: %v", err)
79 }
Michael Anthony Knyszeka97ee2e2023-02-28 19:46:04 +000080 if os.Getenv("GO_BUILDER_NAME") != "" {
81 // On a builder, clean the Go cache in between bent and Sweet.
82 // The build cache can end up using a large portion of the builder's
83 // disk space (~60%), making Sweet run out and fail. Generally speaking
84 // we don't need the build cache because we're going to be doing every
85 // build exactly once from scratch (excluding build benchmarks, which
86 // arrange for a cacheless build their own way). However, we don't want
87 // to do this on a regular development machine because we might want to
88 // run benchmarks with the same toolchain again.
89 //
90 // Note that we only need to clean the cache with one toolchain because
91 // the build cache is shared.
92 if err := cleanGoCache(tcs[0]); err != nil {
93 return fmt.Errorf("failed to clean Go cache: %w", err)
94 }
95 }
Michael Anthony Knyszeke4ea61d2022-01-13 01:22:39 +000096 if err := sweet(tcs); err != nil {
97 pass = false
98 log.Printf("Error running sweet: %v", err)
99 }
Michael Pratt88d4abd2021-10-12 17:44:09 -0400100 if !pass {
Michael Pratt06e21e22022-01-06 18:01:12 -0500101 return fmt.Errorf("benchmarks failed")
102 }
103 return nil
104}
105
Michael Anthony Knyszeka97ee2e2023-02-28 19:46:04 +0000106func cleanGoCache(tc *toolchain) error {
107 wd, err := os.Getwd()
108 if err != nil {
109 return err
110 }
111 if err := tc.Go.Do(wd, "clean", "-cache"); err != nil {
112 return fmt.Errorf("toolchain %s: %w", tc.Name, err)
113 }
114 return nil
115}
116
Michael Pratt06e21e22022-01-06 18:01:12 -0500117func main() {
Michael Pratt4bd3f692022-01-10 12:02:02 -0500118 flag.Parse()
119
120 if *wait {
121 // We may be on a freshly booted VM. Wait for boot tasks to
122 // complete before continuing.
123 if err := waitForIdle(); err != nil {
124 log.Fatalf("Failed to wait for idle: %v", err)
125 }
126 }
127
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +0000128 // Find the toolchain under test.
Michael Anthony Knyszek4fbb60f2022-02-14 22:49:30 +0000129 gorootExperiment := *gorootExperiment
130 if gorootExperiment == "" {
131 var err error
132 gorootExperiment, err = determineGOROOT()
133 if err != nil {
134 log.Fatalf("Unable to determine GOROOT: %v", err)
135 }
Michael Pratt06e21e22022-01-06 18:01:12 -0500136 }
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +0000137 toolchains := []*toolchain{toolchainFromGOROOT("experiment", gorootExperiment)}
Michael Pratt06e21e22022-01-06 18:01:12 -0500138
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +0000139 // Find the baseline toolchain, if applicable.
Michael Anthony Knyszek4fbb60f2022-02-14 22:49:30 +0000140 gorootBaseline := *gorootBaseline
141 if gorootBaseline == "" {
142 gorootBaseline = os.Getenv("BENCH_BASELINE_GOROOT")
143 }
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +0000144 if gorootBaseline != "" {
145 toolchains = append(toolchains, toolchainFromGOROOT("baseline", gorootBaseline))
Michael Pratt06e21e22022-01-06 18:01:12 -0500146 }
147
Michael Prattd4b9fce2022-06-24 13:33:00 -0400148 // Determine the repository we are testing. Defaults to 'go' because
149 // old versions of the coordinator don't specify the repository, but
150 // also only test go.
151 repository := *repository
152 if repository == "" {
153 repository = os.Getenv("BENCH_REPOSITORY")
Michael Anthony Knyszek1caee452022-02-14 20:46:56 +0000154 }
Michael Prattd4b9fce2022-06-24 13:33:00 -0400155 if repository == "" {
156 repository = "go"
Michael Anthony Knyszek4fbb60f2022-02-14 22:49:30 +0000157 }
Michael Prattd4b9fce2022-06-24 13:33:00 -0400158 fmt.Printf("repository: %s\n", repository)
159
160 // Try to identify the branch. If we can't, just make sure we say so
161 // explicitly.
162 branch := *branch
163 if branch == "" {
164 branch = os.Getenv("BENCH_BRANCH")
165 }
166 if branch == "" {
167 branch = "unknown"
168 }
169 fmt.Printf("branch: %s\n", branch)
170
Dung Le6a564b02022-11-30 14:16:16 -0500171 subRepoExperiment := *subRepoExperiment
172 if subRepoExperiment == "" {
173 subRepoExperiment = os.Getenv("BENCH_SUBREPO_PATH")
Michael Prattd4b9fce2022-06-24 13:33:00 -0400174 }
Dung Le6a564b02022-11-30 14:16:16 -0500175 subRepoBaseline := *subRepoBaseline
176 if subRepoBaseline == "" {
177 subRepoBaseline = os.Getenv("BENCH_SUBREPO_BASELINE_PATH")
178 }
Michael Anthony Knyszek1caee452022-02-14 20:46:56 +0000179
Michael Pratt03c43f22022-12-22 10:18:19 -0500180 fmt.Printf("runstamp: %s\n", time.Now().In(time.UTC).Format(time.RFC3339Nano))
181
Dung Le6a564b02022-11-30 14:16:16 -0500182 if repository != "go" {
183 toolchain := toolchainFromGOROOT("baseline", gorootBaseline)
Dung Le6e5e8c22022-12-22 02:38:47 +0700184 if err := goTestSubrepo(toolchain, repository, subRepoBaseline, subRepoExperiment); err != nil {
Robert Findley2652dad2023-03-06 15:41:13 -0500185 log.Printf("Error running subrepo tests: %v", err)
Dung Le6a564b02022-11-30 14:16:16 -0500186 log.Print("FAIL")
187 os.Exit(1)
188 }
189 return
190 }
Michael Anthony Knyszekaf7a6772022-01-12 19:41:13 +0000191 // Run benchmarks against the toolchains.
192 if err := run(toolchains); err != nil {
193 log.Print("FAIL")
Michael Pratt88d4abd2021-10-12 17:44:09 -0400194 os.Exit(1)
195 }
196}