blob: 301c5ec477d06b30f3043945f62746d961172eda [file] [log] [blame]
Jeremy Faller19930862021-02-08 15:04:20 -05001// Copyright 2021 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
David Chasef3eefda2021-05-18 15:01:36 -04005//go:build go1.16
Jeremy Faller19930862021-02-08 15:04:20 -05006// +build go1.16
7
8package main
9
10import (
11 "bufio"
12 "bytes"
13 "fmt"
14 "io"
15 "os"
16 "os/exec"
Jeremy Faller3dedf392021-02-10 13:45:39 -050017 "path"
Jeremy Faller19930862021-02-08 15:04:20 -050018 "runtime"
Jeremy Faller19930862021-02-08 15:04:20 -050019 "strings"
20 "sync"
Michael Prattd3172f22021-11-05 13:21:55 -040021 "time"
Jeremy Faller19930862021-02-08 15:04:20 -050022)
23
24// Configuration is a structure that holds all the variables necessary to
25// initiate a bent run. These structures are read from a .toml file at
26// boot-time.
27type Configuration struct {
28 Name string // Short name used for binary names, mention on command line
29 Root string // Specific Go root to use for this trial
30 BuildFlags []string // BuildFlags supplied to 'go test -c' for building (e.g., "-p 1")
31 AfterBuild []string // Array of commands to run, output of all commands for a configuration (across binaries) is collected in <runstamp>.<config>.<cmd>
32 GcFlags string // GcFlags supplied to 'go test -c' for building
33 GcEnv []string // Environment variables supplied to 'go test -c' for building
34 RunFlags []string // Extra flags passed to the test binary
35 RunEnv []string // Extra environment variables passed to the test binary
36 RunWrapper []string // (Outermost) Command and args to precede whatever the operation is; may fail in the sandbox.
37 Disabled bool // True if this configuration is temporarily disabled
38 buildStats []BenchStat
39 benchWriter *os.File
David Chase1b330552021-10-06 17:31:32 -040040 rootCopy string // The contents of GOROOT are copied here to allow benchmarking of just the test compilation.
Jeremy Faller19930862021-02-08 15:04:20 -050041}
42
David Chase1b330552021-10-06 17:31:32 -040043var dirs *directories // constant across all configurations, useful in other contexts.
44
Jeremy Faller19930862021-02-08 15:04:20 -050045func (c *Configuration) buildBenchName() string {
46 return c.thingBenchName("build")
47}
48
49func (c *Configuration) thingBenchName(suffix string) string {
Jeremy Faller3dedf392021-02-10 13:45:39 -050050 if len(suffix) != 0 {
51 suffix = path.Base(suffix)
Jeremy Faller19930862021-02-08 15:04:20 -050052 }
David Chase1b330552021-10-06 17:31:32 -040053 return path.Join(dirs.benchDir, runstamp+"."+c.Name+"."+suffix)
Jeremy Faller19930862021-02-08 15:04:20 -050054}
55
56func (c *Configuration) benchName(b *Benchmark) string {
57 return b.Name + "_" + c.Name
58}
59
60func (c *Configuration) goCommand() string {
61 gocmd := "go"
62 if c.Root != "" {
Jeremy Faller3dedf392021-02-10 13:45:39 -050063 gocmd = path.Join(c.Root+"bin", gocmd)
Jeremy Faller19930862021-02-08 15:04:20 -050064 }
65 return gocmd
66}
67
68func (c *Configuration) goCommandCopy() string {
69 gocmd := "go"
70 if c.rootCopy != "" {
Jeremy Faller3dedf392021-02-10 13:45:39 -050071 gocmd = path.Join(c.rootCopy, "bin", gocmd)
Jeremy Faller19930862021-02-08 15:04:20 -050072 }
73 return gocmd
74}
75
76func (config *Configuration) createFilesForLater() {
77 if config.Disabled {
78 return
79 }
80 f, err := os.Create(config.buildBenchName())
81 if err != nil {
82 fmt.Println("Error creating build benchmark file ", config.buildBenchName(), ", err=", err)
83 config.Disabled = true
84 } else {
85 fmt.Fprintf(f, "goos: %s\n", runtime.GOOS)
86 fmt.Fprintf(f, "goarch: %s\n", runtime.GOARCH)
87 f.Close() // will be appending later
88 }
89
90 for _, cmd := range config.AfterBuild {
91 tbn := config.thingBenchName(cmd)
92 f, err := os.Create(tbn)
93 if err != nil {
94 fmt.Printf("Error creating %s benchmark file %s, err=%v\n", cmd, config.thingBenchName(cmd), err)
95 continue
96 } else {
97 f.Close() // will be appending later
98 }
99 }
100}
101
102func (config *Configuration) runOtherBenchmarks(b *Benchmark, cwd string) {
103 // Run various other "benchmark" commands on the built binaries, e.g., size, quality of debugging information.
104 if config.Disabled {
105 return
106 }
107
108 for _, cmd := range config.AfterBuild {
109 tbn := config.thingBenchName(cmd)
110 f, err := os.OpenFile(tbn, os.O_WRONLY|os.O_APPEND, os.ModePerm)
111 if err != nil {
112 fmt.Printf("There was an error opening %s for append, error %v\n", tbn, err)
113 continue
114 }
115
116 if !strings.ContainsAny(cmd, "/") {
Jeremy Faller3dedf392021-02-10 13:45:39 -0500117 cmd = path.Join(cwd, cmd)
Jeremy Faller19930862021-02-08 15:04:20 -0500118 }
119 if b.Disabled {
120 continue
121 }
122 testBinaryName := config.benchName(b)
David Chase1b330552021-10-06 17:31:32 -0400123 c := exec.Command(cmd, path.Join(cwd, dirs.testBinDir, testBinaryName), b.Name)
Jeremy Faller19930862021-02-08 15:04:20 -0500124
125 c.Env = defaultEnv
126 if !b.NotSandboxed {
127 c.Env = replaceEnv(c.Env, "GOOS", "linux")
128 }
129 // Match the build environment here.
130 c.Env = replaceEnvs(c.Env, b.GcEnv)
131 c.Env = replaceEnvs(c.Env, config.GcEnv)
132
133 if verbose > 0 {
134 fmt.Println(asCommandLine(cwd, c))
135 }
136 output, err := c.CombinedOutput()
137 if verbose > 0 || err != nil {
138 fmt.Println(string(output))
139 } else {
140 fmt.Print(".")
141 }
142 if err != nil {
143 fmt.Printf("Error running %s\n", cmd)
144 continue
145 }
146 f.Write(output)
147 f.Sync()
148 f.Close()
149 }
150}
151
152func (config *Configuration) compileOne(bench *Benchmark, cwd string, count int) string {
153 root := config.rootCopy
154 gocmd := config.goCommandCopy()
Jeremy Faller3dedf392021-02-10 13:45:39 -0500155 gopath := path.Join(cwd, "gopath")
Jeremy Faller19930862021-02-08 15:04:20 -0500156
157 if explicitAll != 1 { // clear cache unless "-a[=1]" which requests -a on compilation.
158 cmd := exec.Command(gocmd, "clean", "-cache")
159 cmd.Env = defaultEnv
160 if !bench.NotSandboxed {
161 cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
162 }
163 if root != "" {
164 cmd.Env = replaceEnv(cmd.Env, "GOROOT", root)
165 }
166 cmd.Env = replaceEnvs(cmd.Env, bench.GcEnv)
167 cmd.Env = replaceEnvs(cmd.Env, config.GcEnv)
168 cmd.Dir = gopath // Only want the cache-cleaning effect, not the binary-deleting effect. It's okay to clean gopath.
169 s, _ := config.runBinary("", cmd, true)
170 if s != "" {
171 fmt.Println("Error running go clean -cache, ", s)
172 }
173 }
174
Michael Prattd3172f22021-11-05 13:21:55 -0400175 cmd := exec.Command(gocmd, "test", "-vet=off", "-c")
David Chase1b330552021-10-06 17:31:32 -0400176 compileTo := path.Join(dirs.wd, dirs.testBinDir, config.benchName(bench))
177 cmd.Args = append(cmd.Args, "-o", compileTo)
Jeremy Faller19930862021-02-08 15:04:20 -0500178 cmd.Args = append(cmd.Args, bench.BuildFlags...)
179 // Do not normally need -a because cache was emptied first and std was -a installed with these flags.
180 // But for -a=1, do it anyway
181 if explicitAll == 1 {
182 cmd.Args = append(cmd.Args, "-a")
183 }
184 cmd.Args = append(cmd.Args, config.BuildFlags...)
185 if config.GcFlags != "" {
186 cmd.Args = append(cmd.Args, "-gcflags="+config.GcFlags)
187 }
David Chase1b330552021-10-06 17:31:32 -0400188 cmd.Args = append(cmd.Args, bench.Repo)
189 cmd.Dir = dirs.build // use module-mode
Jeremy Faller19930862021-02-08 15:04:20 -0500190 cmd.Env = defaultEnv
191 if !bench.NotSandboxed {
192 cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
193 }
194 if root != "" {
195 cmd.Env = replaceEnv(cmd.Env, "GOROOT", root)
196 }
197 cmd.Env = replaceEnvs(cmd.Env, bench.GcEnv)
198 cmd.Env = replaceEnvs(cmd.Env, config.GcEnv)
199
200 if verbose > 0 {
201 fmt.Println(asCommandLine(cwd, cmd))
202 } else {
203 fmt.Print(".")
204 }
205
206 defer cleanup(gopath)
207
Michael Prattd3172f22021-11-05 13:21:55 -0400208 start := time.Now()
Jeremy Faller19930862021-02-08 15:04:20 -0500209 output, err := cmd.CombinedOutput()
Michael Prattd3172f22021-11-05 13:21:55 -0400210 realTime := time.Since(start)
Jeremy Faller19930862021-02-08 15:04:20 -0500211 if err != nil {
212 s := ""
213 switch e := err.(type) {
214 case *exec.ExitError:
215 s = fmt.Sprintf("There was an error running 'go test', output = %s", output)
216 default:
217 s = fmt.Sprintf("There was an error running 'go test', output = %s, error = %v", output, e)
218 }
219 fmt.Println(s + "DISABLING benchmark " + bench.Name)
220 bench.Disabled = true // if it won't compile, it won't run, either.
221 return s + "(" + bench.Name + ")\n"
222 }
223 soutput := string(output)
Michael Prattd3172f22021-11-05 13:21:55 -0400224 bs := BenchStat{
225 Name: bench.Name,
226 RealTime: realTime,
227 UserTime: cmd.ProcessState.UserTime(),
228 SysTime: cmd.ProcessState.SystemTime(),
229 }
230 config.buildStats = append(config.buildStats, bs)
Jeremy Faller19930862021-02-08 15:04:20 -0500231
232 // Report and record build stats to testbin
233
234 buf := new(bytes.Buffer)
235 configGoArch := getenv(config.GcEnv, "GOARCH")
236 if configGoArch != runtime.GOARCH && configGoArch != "" {
237 s := fmt.Sprintf("goarch: %s-%s\n", runtime.GOARCH, configGoArch)
238 if verbose > 0 {
239 fmt.Print(s)
240 }
241 buf.WriteString(s)
242 }
243 s := fmt.Sprintf("Benchmark%s 1 %d build-real-ns/op %d build-user-ns/op %d build-sys-ns/op\n",
Michael Prattd3172f22021-11-05 13:21:55 -0400244 strings.Title(bench.Name), bs.RealTime.Nanoseconds(), bs.UserTime.Nanoseconds(), bs.SysTime.Nanoseconds())
Jeremy Faller19930862021-02-08 15:04:20 -0500245 if verbose > 0 {
246 fmt.Print(s)
247 }
248 buf.WriteString(s)
249 f, err := os.OpenFile(config.buildBenchName(), os.O_WRONLY|os.O_APPEND, os.ModePerm)
250 if err != nil {
251 fmt.Printf("There was an error opening %s for append, error %v\n", config.buildBenchName(), err)
252 cleanup(gopath)
253 os.Exit(2)
254 }
255 f.Write(buf.Bytes())
256 f.Sync()
257 f.Close()
258
Jeremy Faller19930862021-02-08 15:04:20 -0500259 // Trim /usr/bin/time info from soutput, it's ugly
260 if verbose > 0 {
Jeremy Faller19930862021-02-08 15:04:20 -0500261 i := strings.LastIndex(soutput, "real")
262 if i >= 0 {
263 soutput = soutput[:i]
264 }
265 fmt.Print(soutput)
266 }
267
268 // Do this here before any cleanup.
269 if count == 0 {
270 config.runOtherBenchmarks(bench, cwd)
271 }
272
273 return ""
274}
275
David Chase1b330552021-10-06 17:31:32 -0400276// say writes s to c's benchmark output file
277func (c *Configuration) say(s string) {
278 b := []byte(s)
279 nw, err := c.benchWriter.Write(b)
280 if err != nil {
281 fmt.Printf("Error writing, err = %v, nwritten = %d, nrequested = %d\n", err, nw, len(b))
282 }
283 c.benchWriter.Sync()
284 fmt.Print(string(b))
285}
286
Jeremy Faller19930862021-02-08 15:04:20 -0500287// runBinary runs cmd and displays the output.
288// If the command returns an error, returns an error string.
289func (c *Configuration) runBinary(cwd string, cmd *exec.Cmd, printWorkingDot bool) (string, int) {
290 line := asCommandLine(cwd, cmd)
291 if verbose > 0 {
292 fmt.Println(line)
293 } else {
294 if printWorkingDot {
295 fmt.Print(".")
296 }
297 }
298
299 rc := 0
300
301 stdout, err := cmd.StdoutPipe()
302 if err != nil {
303 return fmt.Sprintf("Error [stdoutpipe] running '%s', %v", line, err), rc
304 }
305 stderr, err := cmd.StderrPipe()
306 if err != nil {
307 return fmt.Sprintf("Error [stderrpipe] running '%s', %v", line, err), rc
308 }
309 err = cmd.Start()
310 if err != nil {
311 return fmt.Sprintf("Error [command start] running '%s', %v", line, err), rc
312 }
313
314 var mu = &sync.Mutex{}
315
316 f := func(r *bufio.Reader, done chan error) {
317 for {
318 bytes, err := r.ReadBytes('\n')
319 n := len(bytes)
320 if n > 0 {
321 mu.Lock()
322 nw, err := c.benchWriter.Write(bytes[0:n])
323 if err != nil {
324 fmt.Printf("Error writing, err = %v, nwritten = %d, nrequested = %d\n", err, nw, n)
325 }
326 c.benchWriter.Sync()
327 fmt.Print(string(bytes[0:n]))
328 mu.Unlock()
329 }
330 if err == io.EOF || n == 0 {
331 break
332 }
333 if err != nil {
334 done <- err
335 return
336 }
337 }
338 done <- nil
339 }
340
341 doneS := make(chan error)
342 doneE := make(chan error)
343
344 go f(bufio.NewReader(stdout), doneS)
345 go f(bufio.NewReader(stderr), doneE)
346
347 errS := <-doneS
348 errE := <-doneE
349
350 err = cmd.Wait()
351 rc = cmd.ProcessState.ExitCode()
352
353 if err != nil {
354 switch e := err.(type) {
355 case *exec.ExitError:
356 return fmt.Sprintf("Error running '%s', stderr = %s, rc = %d", line, e.Stderr, rc), rc
357 default:
358 return fmt.Sprintf("Error running '%s', %v, rc = %d", line, e, rc), rc
359
360 }
361 }
362 if errS != nil {
363 return fmt.Sprintf("Error [read stdout] running '%s', %v, rc = %d", line, errS, rc), rc
364 }
365 if errE != nil {
366 return fmt.Sprintf("Error [read stderr] running '%s', %v, rc = %d", line, errE, rc), rc
367 }
368 return "", rc
369}