Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 1 | // Copyright 2021 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
David Chase | f3eefda | 2021-05-18 15:01:36 -0400 | [diff] [blame] | 5 | //go:build go1.16 |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 6 | // +build go1.16 |
| 7 | |
| 8 | package main |
| 9 | |
| 10 | import ( |
| 11 | "bufio" |
| 12 | "bytes" |
| 13 | "fmt" |
| 14 | "io" |
| 15 | "os" |
| 16 | "os/exec" |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 17 | "path" |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 18 | "runtime" |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 19 | "strings" |
| 20 | "sync" |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 21 | "time" |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 22 | ) |
| 23 | |
| 24 | // Configuration is a structure that holds all the variables necessary to |
| 25 | // initiate a bent run. These structures are read from a .toml file at |
| 26 | // boot-time. |
| 27 | type Configuration struct { |
| 28 | Name string // Short name used for binary names, mention on command line |
| 29 | Root string // Specific Go root to use for this trial |
| 30 | BuildFlags []string // BuildFlags supplied to 'go test -c' for building (e.g., "-p 1") |
| 31 | AfterBuild []string // Array of commands to run, output of all commands for a configuration (across binaries) is collected in <runstamp>.<config>.<cmd> |
| 32 | GcFlags string // GcFlags supplied to 'go test -c' for building |
| 33 | GcEnv []string // Environment variables supplied to 'go test -c' for building |
| 34 | RunFlags []string // Extra flags passed to the test binary |
| 35 | RunEnv []string // Extra environment variables passed to the test binary |
| 36 | RunWrapper []string // (Outermost) Command and args to precede whatever the operation is; may fail in the sandbox. |
| 37 | Disabled bool // True if this configuration is temporarily disabled |
| 38 | buildStats []BenchStat |
| 39 | benchWriter *os.File |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 40 | rootCopy string // The contents of GOROOT are copied here to allow benchmarking of just the test compilation. |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 41 | } |
| 42 | |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 43 | var dirs *directories // constant across all configurations, useful in other contexts. |
| 44 | |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 45 | func (c *Configuration) buildBenchName() string { |
| 46 | return c.thingBenchName("build") |
| 47 | } |
| 48 | |
| 49 | func (c *Configuration) thingBenchName(suffix string) string { |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 50 | if len(suffix) != 0 { |
| 51 | suffix = path.Base(suffix) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 52 | } |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 53 | return path.Join(dirs.benchDir, runstamp+"."+c.Name+"."+suffix) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | func (c *Configuration) benchName(b *Benchmark) string { |
| 57 | return b.Name + "_" + c.Name |
| 58 | } |
| 59 | |
| 60 | func (c *Configuration) goCommand() string { |
| 61 | gocmd := "go" |
| 62 | if c.Root != "" { |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 63 | gocmd = path.Join(c.Root+"bin", gocmd) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 64 | } |
| 65 | return gocmd |
| 66 | } |
| 67 | |
| 68 | func (c *Configuration) goCommandCopy() string { |
| 69 | gocmd := "go" |
| 70 | if c.rootCopy != "" { |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 71 | gocmd = path.Join(c.rootCopy, "bin", gocmd) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 72 | } |
| 73 | return gocmd |
| 74 | } |
| 75 | |
| 76 | func (config *Configuration) createFilesForLater() { |
| 77 | if config.Disabled { |
| 78 | return |
| 79 | } |
| 80 | f, err := os.Create(config.buildBenchName()) |
| 81 | if err != nil { |
| 82 | fmt.Println("Error creating build benchmark file ", config.buildBenchName(), ", err=", err) |
| 83 | config.Disabled = true |
| 84 | } else { |
| 85 | fmt.Fprintf(f, "goos: %s\n", runtime.GOOS) |
| 86 | fmt.Fprintf(f, "goarch: %s\n", runtime.GOARCH) |
| 87 | f.Close() // will be appending later |
| 88 | } |
| 89 | |
| 90 | for _, cmd := range config.AfterBuild { |
| 91 | tbn := config.thingBenchName(cmd) |
| 92 | f, err := os.Create(tbn) |
| 93 | if err != nil { |
| 94 | fmt.Printf("Error creating %s benchmark file %s, err=%v\n", cmd, config.thingBenchName(cmd), err) |
| 95 | continue |
| 96 | } else { |
| 97 | f.Close() // will be appending later |
| 98 | } |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | func (config *Configuration) runOtherBenchmarks(b *Benchmark, cwd string) { |
| 103 | // Run various other "benchmark" commands on the built binaries, e.g., size, quality of debugging information. |
| 104 | if config.Disabled { |
| 105 | return |
| 106 | } |
| 107 | |
| 108 | for _, cmd := range config.AfterBuild { |
| 109 | tbn := config.thingBenchName(cmd) |
| 110 | f, err := os.OpenFile(tbn, os.O_WRONLY|os.O_APPEND, os.ModePerm) |
| 111 | if err != nil { |
| 112 | fmt.Printf("There was an error opening %s for append, error %v\n", tbn, err) |
| 113 | continue |
| 114 | } |
| 115 | |
| 116 | if !strings.ContainsAny(cmd, "/") { |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 117 | cmd = path.Join(cwd, cmd) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 118 | } |
| 119 | if b.Disabled { |
| 120 | continue |
| 121 | } |
| 122 | testBinaryName := config.benchName(b) |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 123 | c := exec.Command(cmd, path.Join(cwd, dirs.testBinDir, testBinaryName), b.Name) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 124 | |
| 125 | c.Env = defaultEnv |
| 126 | if !b.NotSandboxed { |
| 127 | c.Env = replaceEnv(c.Env, "GOOS", "linux") |
| 128 | } |
| 129 | // Match the build environment here. |
| 130 | c.Env = replaceEnvs(c.Env, b.GcEnv) |
| 131 | c.Env = replaceEnvs(c.Env, config.GcEnv) |
| 132 | |
| 133 | if verbose > 0 { |
| 134 | fmt.Println(asCommandLine(cwd, c)) |
| 135 | } |
| 136 | output, err := c.CombinedOutput() |
| 137 | if verbose > 0 || err != nil { |
| 138 | fmt.Println(string(output)) |
| 139 | } else { |
| 140 | fmt.Print(".") |
| 141 | } |
| 142 | if err != nil { |
| 143 | fmt.Printf("Error running %s\n", cmd) |
| 144 | continue |
| 145 | } |
| 146 | f.Write(output) |
| 147 | f.Sync() |
| 148 | f.Close() |
| 149 | } |
| 150 | } |
| 151 | |
| 152 | func (config *Configuration) compileOne(bench *Benchmark, cwd string, count int) string { |
| 153 | root := config.rootCopy |
| 154 | gocmd := config.goCommandCopy() |
Jeremy Faller | 3dedf39 | 2021-02-10 13:45:39 -0500 | [diff] [blame] | 155 | gopath := path.Join(cwd, "gopath") |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 156 | |
| 157 | if explicitAll != 1 { // clear cache unless "-a[=1]" which requests -a on compilation. |
| 158 | cmd := exec.Command(gocmd, "clean", "-cache") |
| 159 | cmd.Env = defaultEnv |
| 160 | if !bench.NotSandboxed { |
| 161 | cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux") |
| 162 | } |
| 163 | if root != "" { |
| 164 | cmd.Env = replaceEnv(cmd.Env, "GOROOT", root) |
| 165 | } |
| 166 | cmd.Env = replaceEnvs(cmd.Env, bench.GcEnv) |
| 167 | cmd.Env = replaceEnvs(cmd.Env, config.GcEnv) |
| 168 | cmd.Dir = gopath // Only want the cache-cleaning effect, not the binary-deleting effect. It's okay to clean gopath. |
| 169 | s, _ := config.runBinary("", cmd, true) |
| 170 | if s != "" { |
| 171 | fmt.Println("Error running go clean -cache, ", s) |
| 172 | } |
| 173 | } |
| 174 | |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 175 | cmd := exec.Command(gocmd, "test", "-vet=off", "-c") |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 176 | compileTo := path.Join(dirs.wd, dirs.testBinDir, config.benchName(bench)) |
| 177 | cmd.Args = append(cmd.Args, "-o", compileTo) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 178 | cmd.Args = append(cmd.Args, bench.BuildFlags...) |
| 179 | // Do not normally need -a because cache was emptied first and std was -a installed with these flags. |
| 180 | // But for -a=1, do it anyway |
| 181 | if explicitAll == 1 { |
| 182 | cmd.Args = append(cmd.Args, "-a") |
| 183 | } |
| 184 | cmd.Args = append(cmd.Args, config.BuildFlags...) |
| 185 | if config.GcFlags != "" { |
| 186 | cmd.Args = append(cmd.Args, "-gcflags="+config.GcFlags) |
| 187 | } |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 188 | cmd.Args = append(cmd.Args, bench.Repo) |
| 189 | cmd.Dir = dirs.build // use module-mode |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 190 | cmd.Env = defaultEnv |
| 191 | if !bench.NotSandboxed { |
| 192 | cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux") |
| 193 | } |
| 194 | if root != "" { |
| 195 | cmd.Env = replaceEnv(cmd.Env, "GOROOT", root) |
| 196 | } |
| 197 | cmd.Env = replaceEnvs(cmd.Env, bench.GcEnv) |
| 198 | cmd.Env = replaceEnvs(cmd.Env, config.GcEnv) |
| 199 | |
| 200 | if verbose > 0 { |
| 201 | fmt.Println(asCommandLine(cwd, cmd)) |
| 202 | } else { |
| 203 | fmt.Print(".") |
| 204 | } |
| 205 | |
| 206 | defer cleanup(gopath) |
| 207 | |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 208 | start := time.Now() |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 209 | output, err := cmd.CombinedOutput() |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 210 | realTime := time.Since(start) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 211 | if err != nil { |
| 212 | s := "" |
| 213 | switch e := err.(type) { |
| 214 | case *exec.ExitError: |
| 215 | s = fmt.Sprintf("There was an error running 'go test', output = %s", output) |
| 216 | default: |
| 217 | s = fmt.Sprintf("There was an error running 'go test', output = %s, error = %v", output, e) |
| 218 | } |
| 219 | fmt.Println(s + "DISABLING benchmark " + bench.Name) |
| 220 | bench.Disabled = true // if it won't compile, it won't run, either. |
| 221 | return s + "(" + bench.Name + ")\n" |
| 222 | } |
| 223 | soutput := string(output) |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 224 | bs := BenchStat{ |
| 225 | Name: bench.Name, |
| 226 | RealTime: realTime, |
| 227 | UserTime: cmd.ProcessState.UserTime(), |
| 228 | SysTime: cmd.ProcessState.SystemTime(), |
| 229 | } |
| 230 | config.buildStats = append(config.buildStats, bs) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 231 | |
| 232 | // Report and record build stats to testbin |
| 233 | |
| 234 | buf := new(bytes.Buffer) |
| 235 | configGoArch := getenv(config.GcEnv, "GOARCH") |
| 236 | if configGoArch != runtime.GOARCH && configGoArch != "" { |
| 237 | s := fmt.Sprintf("goarch: %s-%s\n", runtime.GOARCH, configGoArch) |
| 238 | if verbose > 0 { |
| 239 | fmt.Print(s) |
| 240 | } |
| 241 | buf.WriteString(s) |
| 242 | } |
| 243 | s := fmt.Sprintf("Benchmark%s 1 %d build-real-ns/op %d build-user-ns/op %d build-sys-ns/op\n", |
Michael Pratt | d3172f2 | 2021-11-05 13:21:55 -0400 | [diff] [blame^] | 244 | strings.Title(bench.Name), bs.RealTime.Nanoseconds(), bs.UserTime.Nanoseconds(), bs.SysTime.Nanoseconds()) |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 245 | if verbose > 0 { |
| 246 | fmt.Print(s) |
| 247 | } |
| 248 | buf.WriteString(s) |
| 249 | f, err := os.OpenFile(config.buildBenchName(), os.O_WRONLY|os.O_APPEND, os.ModePerm) |
| 250 | if err != nil { |
| 251 | fmt.Printf("There was an error opening %s for append, error %v\n", config.buildBenchName(), err) |
| 252 | cleanup(gopath) |
| 253 | os.Exit(2) |
| 254 | } |
| 255 | f.Write(buf.Bytes()) |
| 256 | f.Sync() |
| 257 | f.Close() |
| 258 | |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 259 | // Trim /usr/bin/time info from soutput, it's ugly |
| 260 | if verbose > 0 { |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 261 | i := strings.LastIndex(soutput, "real") |
| 262 | if i >= 0 { |
| 263 | soutput = soutput[:i] |
| 264 | } |
| 265 | fmt.Print(soutput) |
| 266 | } |
| 267 | |
| 268 | // Do this here before any cleanup. |
| 269 | if count == 0 { |
| 270 | config.runOtherBenchmarks(bench, cwd) |
| 271 | } |
| 272 | |
| 273 | return "" |
| 274 | } |
| 275 | |
David Chase | 1b33055 | 2021-10-06 17:31:32 -0400 | [diff] [blame] | 276 | // say writes s to c's benchmark output file |
| 277 | func (c *Configuration) say(s string) { |
| 278 | b := []byte(s) |
| 279 | nw, err := c.benchWriter.Write(b) |
| 280 | if err != nil { |
| 281 | fmt.Printf("Error writing, err = %v, nwritten = %d, nrequested = %d\n", err, nw, len(b)) |
| 282 | } |
| 283 | c.benchWriter.Sync() |
| 284 | fmt.Print(string(b)) |
| 285 | } |
| 286 | |
Jeremy Faller | 1993086 | 2021-02-08 15:04:20 -0500 | [diff] [blame] | 287 | // runBinary runs cmd and displays the output. |
| 288 | // If the command returns an error, returns an error string. |
| 289 | func (c *Configuration) runBinary(cwd string, cmd *exec.Cmd, printWorkingDot bool) (string, int) { |
| 290 | line := asCommandLine(cwd, cmd) |
| 291 | if verbose > 0 { |
| 292 | fmt.Println(line) |
| 293 | } else { |
| 294 | if printWorkingDot { |
| 295 | fmt.Print(".") |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | rc := 0 |
| 300 | |
| 301 | stdout, err := cmd.StdoutPipe() |
| 302 | if err != nil { |
| 303 | return fmt.Sprintf("Error [stdoutpipe] running '%s', %v", line, err), rc |
| 304 | } |
| 305 | stderr, err := cmd.StderrPipe() |
| 306 | if err != nil { |
| 307 | return fmt.Sprintf("Error [stderrpipe] running '%s', %v", line, err), rc |
| 308 | } |
| 309 | err = cmd.Start() |
| 310 | if err != nil { |
| 311 | return fmt.Sprintf("Error [command start] running '%s', %v", line, err), rc |
| 312 | } |
| 313 | |
| 314 | var mu = &sync.Mutex{} |
| 315 | |
| 316 | f := func(r *bufio.Reader, done chan error) { |
| 317 | for { |
| 318 | bytes, err := r.ReadBytes('\n') |
| 319 | n := len(bytes) |
| 320 | if n > 0 { |
| 321 | mu.Lock() |
| 322 | nw, err := c.benchWriter.Write(bytes[0:n]) |
| 323 | if err != nil { |
| 324 | fmt.Printf("Error writing, err = %v, nwritten = %d, nrequested = %d\n", err, nw, n) |
| 325 | } |
| 326 | c.benchWriter.Sync() |
| 327 | fmt.Print(string(bytes[0:n])) |
| 328 | mu.Unlock() |
| 329 | } |
| 330 | if err == io.EOF || n == 0 { |
| 331 | break |
| 332 | } |
| 333 | if err != nil { |
| 334 | done <- err |
| 335 | return |
| 336 | } |
| 337 | } |
| 338 | done <- nil |
| 339 | } |
| 340 | |
| 341 | doneS := make(chan error) |
| 342 | doneE := make(chan error) |
| 343 | |
| 344 | go f(bufio.NewReader(stdout), doneS) |
| 345 | go f(bufio.NewReader(stderr), doneE) |
| 346 | |
| 347 | errS := <-doneS |
| 348 | errE := <-doneE |
| 349 | |
| 350 | err = cmd.Wait() |
| 351 | rc = cmd.ProcessState.ExitCode() |
| 352 | |
| 353 | if err != nil { |
| 354 | switch e := err.(type) { |
| 355 | case *exec.ExitError: |
| 356 | return fmt.Sprintf("Error running '%s', stderr = %s, rc = %d", line, e.Stderr, rc), rc |
| 357 | default: |
| 358 | return fmt.Sprintf("Error running '%s', %v, rc = %d", line, e, rc), rc |
| 359 | |
| 360 | } |
| 361 | } |
| 362 | if errS != nil { |
| 363 | return fmt.Sprintf("Error [read stdout] running '%s', %v, rc = %d", line, errS, rc), rc |
| 364 | } |
| 365 | if errE != nil { |
| 366 | return fmt.Sprintf("Error [read stderr] running '%s', %v, rc = %d", line, errE, rc), rc |
| 367 | } |
| 368 | return "", rc |
| 369 | } |