| // Copyright 2015 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Compilebench benchmarks the speed of the Go compiler. |
| // |
| // Usage: |
| // |
| // compilebench [options] |
| // |
| // It times the compilation of various packages and prints results in |
| // the format used by package testing (and expected by golang.org/x/perf/cmd/benchstat). |
| // |
| // The options are: |
| // |
| // -alloc |
| // Report allocations. |
| // |
| // -compile exe |
| // Use exe as the path to the cmd/compile binary. |
| // |
| // -compileflags 'list' |
| // Pass the space-separated list of flags to the compilation. |
| // |
| // -link exe |
| // Use exe as the path to the cmd/link binary. |
| // |
| // -linkflags 'list' |
| // Pass the space-separated list of flags to the linker. |
| // |
| // -count n |
| // Run each benchmark n times (default 1). |
| // |
| // -cpuprofile file |
| // Write a CPU profile of the compiler to file. |
| // |
| // -go path |
| // Path to "go" command (default "go"). |
| // |
| // -memprofile file |
| // Write a memory profile of the compiler to file. |
| // |
| // -memprofilerate rate |
| // Set runtime.MemProfileRate during compilation. |
| // |
| // -obj |
| // Report object file statistics. |
| // |
| // -pkg pkg |
| // Benchmark compiling a single package. |
| // |
| // -run regexp |
| // Only run benchmarks with names matching regexp. |
| // |
| // -short |
| // Skip long-running benchmarks. |
| // |
| // Although -cpuprofile and -memprofile are intended to write a |
| // combined profile for all the executed benchmarks to file, |
| // today they write only the profile for the last benchmark executed. |
| // |
| // The default memory profiling rate is one profile sample per 512 kB |
| // allocated (see “go doc runtime.MemProfileRate”). |
| // Lowering the rate (for example, -memprofilerate 64000) produces |
| // a more fine-grained and therefore accurate profile, but it also incurs |
| // execution cost. For benchmark comparisons, never use timings |
| // obtained with a low -memprofilerate option. |
| // |
| // # Example |
| // |
| // Assuming the base version of the compiler has been saved with |
| // “toolstash save,” this sequence compares the old and new compiler: |
| // |
| // compilebench -count 10 -compile $(toolstash -n compile) >old.txt |
| // compilebench -count 10 >new.txt |
| // benchstat old.txt new.txt |
| package main |
| |
| import ( |
| "bytes" |
| "encoding/json" |
| "flag" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "os" |
| "path/filepath" |
| "regexp" |
| "runtime" |
| "strconv" |
| "strings" |
| "time" |
| |
| exec "golang.org/x/sys/execabs" |
| ) |
| |
| var ( |
| goroot string |
| compiler string |
| assembler string |
| linker string |
| runRE *regexp.Regexp |
| is6g bool |
| ) |
| |
| var ( |
| flagGoCmd = flag.String("go", "go", "path to \"go\" command") |
| flagAlloc = flag.Bool("alloc", false, "report allocations") |
| flagObj = flag.Bool("obj", false, "report object file stats") |
| flagCompiler = flag.String("compile", "", "use `exe` as the cmd/compile binary") |
| flagAssembler = flag.String("asm", "", "use `exe` as the cmd/asm binary") |
| flagCompilerFlags = flag.String("compileflags", "", "additional `flags` to pass to compile") |
| flagLinker = flag.String("link", "", "use `exe` as the cmd/link binary") |
| flagLinkerFlags = flag.String("linkflags", "", "additional `flags` to pass to link") |
| flagRun = flag.String("run", "", "run benchmarks matching `regexp`") |
| flagCount = flag.Int("count", 1, "run benchmarks `n` times") |
| flagCpuprofile = flag.String("cpuprofile", "", "write CPU profile to `file`") |
| flagMemprofile = flag.String("memprofile", "", "write memory profile to `file`") |
| flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`") |
| flagPackage = flag.String("pkg", "", "if set, benchmark the package at path `pkg`") |
| flagShort = flag.Bool("short", false, "skip long-running benchmarks") |
| flagTrace = flag.Bool("trace", false, "debug tracing of builds") |
| ) |
| |
| type test struct { |
| name string |
| r runner |
| } |
| |
| type runner interface { |
| long() bool |
| run(name string, count int) error |
| } |
| |
| var tests = []test{ |
| {"BenchmarkTemplate", compile{"html/template"}}, |
| {"BenchmarkUnicode", compile{"unicode"}}, |
| {"BenchmarkGoTypes", compile{"go/types"}}, |
| {"BenchmarkCompiler", compile{"cmd/compile/internal/gc"}}, |
| {"BenchmarkSSA", compile{"cmd/compile/internal/ssa"}}, |
| {"BenchmarkFlate", compile{"compress/flate"}}, |
| {"BenchmarkGoParser", compile{"go/parser"}}, |
| {"BenchmarkReflect", compile{"reflect"}}, |
| {"BenchmarkTar", compile{"archive/tar"}}, |
| {"BenchmarkXML", compile{"encoding/xml"}}, |
| {"BenchmarkLinkCompiler", link{"cmd/compile", ""}}, |
| {"BenchmarkExternalLinkCompiler", link{"cmd/compile", "-linkmode=external"}}, |
| {"BenchmarkLinkWithoutDebugCompiler", link{"cmd/compile", "-w"}}, |
| {"BenchmarkStdCmd", goBuild{[]string{"std", "cmd"}}}, |
| {"BenchmarkHelloSize", size{"$GOROOT/test/helloworld.go", false}}, |
| {"BenchmarkCmdGoSize", size{"cmd/go", true}}, |
| } |
| |
| func usage() { |
| fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n") |
| fmt.Fprintf(os.Stderr, "options:\n") |
| flag.PrintDefaults() |
| os.Exit(2) |
| } |
| |
| func main() { |
| log.SetFlags(0) |
| log.SetPrefix("compilebench: ") |
| flag.Usage = usage |
| flag.Parse() |
| if flag.NArg() != 0 { |
| usage() |
| } |
| |
| s, err := exec.Command(*flagGoCmd, "env", "GOROOT").CombinedOutput() |
| if err != nil { |
| log.Fatalf("%s env GOROOT: %v", *flagGoCmd, err) |
| } |
| goroot = strings.TrimSpace(string(s)) |
| os.Setenv("GOROOT", goroot) // for any subcommands |
| |
| compiler = *flagCompiler |
| if compiler == "" { |
| var foundTool string |
| foundTool, compiler = toolPath("compile", "6g") |
| if foundTool == "6g" { |
| is6g = true |
| } |
| } |
| assembler = *flagAssembler |
| if assembler == "" { |
| _, assembler = toolPath("asm") |
| } |
| |
| linker = *flagLinker |
| if linker == "" && !is6g { // TODO: Support 6l |
| _, linker = toolPath("link") |
| } |
| |
| if is6g { |
| *flagMemprofilerate = -1 |
| *flagAlloc = false |
| *flagCpuprofile = "" |
| *flagMemprofile = "" |
| } |
| |
| if *flagRun != "" { |
| r, err := regexp.Compile(*flagRun) |
| if err != nil { |
| log.Fatalf("invalid -run argument: %v", err) |
| } |
| runRE = r |
| } |
| |
| if *flagPackage != "" { |
| tests = []test{ |
| {"BenchmarkPkg", compile{*flagPackage}}, |
| {"BenchmarkPkgLink", link{*flagPackage, ""}}, |
| } |
| runRE = nil |
| } |
| |
| for i := 0; i < *flagCount; i++ { |
| for _, tt := range tests { |
| if tt.r.long() && *flagShort { |
| continue |
| } |
| if runRE == nil || runRE.MatchString(tt.name) { |
| if err := tt.r.run(tt.name, i); err != nil { |
| log.Printf("%s: %v", tt.name, err) |
| } |
| } |
| } |
| } |
| } |
| |
| func toolPath(names ...string) (found, path string) { |
| var out1 []byte |
| var err1 error |
| for i, name := range names { |
| out, err := exec.Command(*flagGoCmd, "tool", "-n", name).CombinedOutput() |
| if err == nil { |
| return name, strings.TrimSpace(string(out)) |
| } |
| if i == 0 { |
| out1, err1 = out, err |
| } |
| } |
| log.Fatalf("go tool -n %s: %v\n%s", names[0], err1, out1) |
| return "", "" |
| } |
| |
| type Pkg struct { |
| ImportPath string |
| Dir string |
| GoFiles []string |
| SFiles []string |
| } |
| |
| func goList(dir string) (*Pkg, error) { |
| var pkg Pkg |
| out, err := exec.Command(*flagGoCmd, "list", "-json", dir).Output() |
| if err != nil { |
| return nil, fmt.Errorf("go list -json %s: %v", dir, err) |
| } |
| if err := json.Unmarshal(out, &pkg); err != nil { |
| return nil, fmt.Errorf("go list -json %s: unmarshal: %v", dir, err) |
| } |
| return &pkg, nil |
| } |
| |
| func runCmd(name string, cmd *exec.Cmd) error { |
| start := time.Now() |
| out, err := cmd.CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("%v\n%s", err, out) |
| } |
| fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds()) |
| return nil |
| } |
| |
| type goBuild struct{ pkgs []string } |
| |
| func (goBuild) long() bool { return true } |
| |
| func (r goBuild) run(name string, count int) error { |
| args := []string{"build", "-a"} |
| if *flagCompilerFlags != "" { |
| args = append(args, "-gcflags", *flagCompilerFlags) |
| } |
| args = append(args, r.pkgs...) |
| cmd := exec.Command(*flagGoCmd, args...) |
| cmd.Dir = filepath.Join(goroot, "src") |
| return runCmd(name, cmd) |
| } |
| |
| type size struct { |
| // path is either a path to a file ("$GOROOT/test/helloworld.go") or a package path ("cmd/go"). |
| path string |
| isLong bool |
| } |
| |
| func (r size) long() bool { return r.isLong } |
| |
| func (r size) run(name string, count int) error { |
| if strings.HasPrefix(r.path, "$GOROOT/") { |
| r.path = goroot + "/" + r.path[len("$GOROOT/"):] |
| } |
| |
| cmd := exec.Command(*flagGoCmd, "build", "-o", "_compilebenchout_", r.path) |
| cmd.Stdout = os.Stderr |
| cmd.Stderr = os.Stderr |
| if err := cmd.Run(); err != nil { |
| return err |
| } |
| defer os.Remove("_compilebenchout_") |
| info, err := os.Stat("_compilebenchout_") |
| if err != nil { |
| return err |
| } |
| out, err := exec.Command("size", "_compilebenchout_").CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("size: %v\n%s", err, out) |
| } |
| lines := strings.Split(string(out), "\n") |
| if len(lines) < 2 { |
| return fmt.Errorf("not enough output from size: %s", out) |
| } |
| f := strings.Fields(lines[1]) |
| if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X |
| fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size()) |
| } else if strings.Contains(lines[0], "bss") && len(f) >= 3 { |
| fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size()) |
| } |
| return nil |
| } |
| |
| type compile struct{ dir string } |
| |
| func (compile) long() bool { return false } |
| |
| func (c compile) run(name string, count int) error { |
| // Make sure dependencies needed by go tool compile are installed to GOROOT/pkg. |
| out, err := exec.Command(*flagGoCmd, "build", "-i", c.dir).CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("go build -i %s: %v\n%s", c.dir, err, out) |
| } |
| |
| // Find dir and source file list. |
| pkg, err := goList(c.dir) |
| if err != nil { |
| return err |
| } |
| |
| // If this package has assembly files, we'll need to pass a symabis |
| // file to the compiler; call a helper to invoke the assembler |
| // to do that. |
| var symAbisFile string |
| var asmIncFile string |
| if len(pkg.SFiles) != 0 { |
| symAbisFile = filepath.Join(pkg.Dir, "symabis") |
| asmIncFile = filepath.Join(pkg.Dir, "go_asm.h") |
| content := "\n" |
| if err := os.WriteFile(asmIncFile, []byte(content), 0666); err != nil { |
| return fmt.Errorf("os.WriteFile(%s) failed: %v", asmIncFile, err) |
| } |
| defer os.Remove(symAbisFile) |
| defer os.Remove(asmIncFile) |
| if err := genSymAbisFile(pkg, symAbisFile, pkg.Dir); err != nil { |
| return err |
| } |
| } |
| |
| args := []string{"-o", "_compilebench_.o", "-p", pkg.ImportPath} |
| args = append(args, strings.Fields(*flagCompilerFlags)...) |
| if symAbisFile != "" { |
| args = append(args, "-symabis", symAbisFile) |
| } |
| args = append(args, pkg.GoFiles...) |
| if err := runBuildCmd(name, count, pkg.Dir, compiler, args); err != nil { |
| return err |
| } |
| |
| opath := pkg.Dir + "/_compilebench_.o" |
| if *flagObj { |
| // TODO(josharian): object files are big; just read enough to find what we seek. |
| data, err := ioutil.ReadFile(opath) |
| if err != nil { |
| log.Print(err) |
| } |
| // Find start of export data. |
| i := bytes.Index(data, []byte("\n$$B\n")) + len("\n$$B\n") |
| // Count bytes to end of export data. |
| nexport := bytes.Index(data[i:], []byte("\n$$\n")) |
| fmt.Printf(" %d object-bytes %d export-bytes", len(data), nexport) |
| } |
| fmt.Println() |
| |
| os.Remove(opath) |
| return nil |
| } |
| |
| type link struct{ dir, flags string } |
| |
| func (link) long() bool { return false } |
| |
| func (r link) run(name string, count int) error { |
| if linker == "" { |
| // No linker. Skip the test. |
| return nil |
| } |
| |
| // Build dependencies. |
| out, err := exec.Command(*flagGoCmd, "build", "-i", "-o", "/dev/null", r.dir).CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("go build -i %s: %v\n%s", r.dir, err, out) |
| } |
| |
| // Build the main package. |
| pkg, err := goList(r.dir) |
| if err != nil { |
| return err |
| } |
| args := []string{"-o", "_compilebench_.o"} |
| args = append(args, pkg.GoFiles...) |
| cmd := exec.Command(compiler, args...) |
| cmd.Dir = pkg.Dir |
| cmd.Stdout = os.Stderr |
| cmd.Stderr = os.Stderr |
| err = cmd.Run() |
| if err != nil { |
| return fmt.Errorf("compiling: %v", err) |
| } |
| defer os.Remove(pkg.Dir + "/_compilebench_.o") |
| |
| // Link the main package. |
| args = []string{"-o", "_compilebench_.exe"} |
| args = append(args, strings.Fields(*flagLinkerFlags)...) |
| args = append(args, strings.Fields(r.flags)...) |
| args = append(args, "_compilebench_.o") |
| if err := runBuildCmd(name, count, pkg.Dir, linker, args); err != nil { |
| return err |
| } |
| fmt.Println() |
| defer os.Remove(pkg.Dir + "/_compilebench_.exe") |
| |
| return err |
| } |
| |
| // runBuildCmd runs "tool args..." in dir, measures standard build |
| // tool metrics, and prints a benchmark line. The caller may print |
| // additional metrics and then must print a newline. |
| // |
| // This assumes tool accepts standard build tool flags like |
| // -memprofilerate, -memprofile, and -cpuprofile. |
| func runBuildCmd(name string, count int, dir, tool string, args []string) error { |
| var preArgs []string |
| if *flagMemprofilerate >= 0 { |
| preArgs = append(preArgs, "-memprofilerate", fmt.Sprint(*flagMemprofilerate)) |
| } |
| if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" { |
| if *flagAlloc || *flagMemprofile != "" { |
| preArgs = append(preArgs, "-memprofile", "_compilebench_.memprof") |
| } |
| if *flagCpuprofile != "" { |
| preArgs = append(preArgs, "-cpuprofile", "_compilebench_.cpuprof") |
| } |
| } |
| if *flagTrace { |
| fmt.Fprintf(os.Stderr, "running: %s %+v\n", |
| tool, append(preArgs, args...)) |
| } |
| cmd := exec.Command(tool, append(preArgs, args...)...) |
| cmd.Dir = dir |
| cmd.Stdout = os.Stderr |
| cmd.Stderr = os.Stderr |
| start := time.Now() |
| err := cmd.Run() |
| if err != nil { |
| return err |
| } |
| end := time.Now() |
| |
| haveAllocs, haveRSS := false, false |
| var allocs, allocbytes, rssbytes int64 |
| if *flagAlloc || *flagMemprofile != "" { |
| out, err := ioutil.ReadFile(dir + "/_compilebench_.memprof") |
| if err != nil { |
| log.Print("cannot find memory profile after compilation") |
| } |
| for _, line := range strings.Split(string(out), "\n") { |
| f := strings.Fields(line) |
| if len(f) < 4 || f[0] != "#" || f[2] != "=" { |
| continue |
| } |
| val, err := strconv.ParseInt(f[3], 0, 64) |
| if err != nil { |
| continue |
| } |
| haveAllocs = true |
| switch f[1] { |
| case "TotalAlloc": |
| allocbytes = val |
| case "Mallocs": |
| allocs = val |
| case "MaxRSS": |
| haveRSS = true |
| rssbytes = val |
| } |
| } |
| if !haveAllocs { |
| log.Println("missing stats in memprof (golang.org/issue/18641)") |
| } |
| |
| if *flagMemprofile != "" { |
| outpath := *flagMemprofile |
| if *flagCount != 1 { |
| outpath = fmt.Sprintf("%s_%d", outpath, count) |
| } |
| if err := ioutil.WriteFile(outpath, out, 0666); err != nil { |
| log.Print(err) |
| } |
| } |
| os.Remove(dir + "/_compilebench_.memprof") |
| } |
| |
| if *flagCpuprofile != "" { |
| out, err := ioutil.ReadFile(dir + "/_compilebench_.cpuprof") |
| if err != nil { |
| log.Print(err) |
| } |
| outpath := *flagCpuprofile |
| if *flagCount != 1 { |
| outpath = fmt.Sprintf("%s_%d", outpath, count) |
| } |
| if err := ioutil.WriteFile(outpath, out, 0666); err != nil { |
| log.Print(err) |
| } |
| os.Remove(dir + "/_compilebench_.cpuprof") |
| } |
| |
| wallns := end.Sub(start).Nanoseconds() |
| userns := cmd.ProcessState.UserTime().Nanoseconds() |
| |
| fmt.Printf("%s 1 %d ns/op %d user-ns/op", name, wallns, userns) |
| if haveAllocs { |
| fmt.Printf(" %d B/op %d allocs/op", allocbytes, allocs) |
| } |
| if haveRSS { |
| fmt.Printf(" %d maxRSS/op", rssbytes) |
| } |
| |
| return nil |
| } |
| |
| // genSymAbisFile runs the assembler on the target packge asm files |
| // with "-gensymabis" to produce a symabis file that will feed into |
| // the Go source compilation. This is fairly hacky in that if the |
| // asm invocation convenion changes it will need to be updated |
| // (hopefully that will not be needed too frequently). |
| func genSymAbisFile(pkg *Pkg, symAbisFile, incdir string) error { |
| args := []string{"-gensymabis", "-o", symAbisFile, |
| "-p", pkg.ImportPath, |
| "-I", filepath.Join(goroot, "pkg", "include"), |
| "-I", incdir, |
| "-D", "GOOS_" + runtime.GOOS, |
| "-D", "GOARCH_" + runtime.GOARCH} |
| if pkg.ImportPath == "reflect" { |
| args = append(args, "-compiling-runtime") |
| } |
| args = append(args, pkg.SFiles...) |
| if *flagTrace { |
| fmt.Fprintf(os.Stderr, "running: %s %+v\n", |
| assembler, args) |
| } |
| cmd := exec.Command(assembler, args...) |
| cmd.Dir = pkg.Dir |
| cmd.Stdout = os.Stderr |
| cmd.Stderr = os.Stderr |
| err := cmd.Run() |
| if err != nil { |
| return fmt.Errorf("assembling to produce symabis file: %v", err) |
| } |
| return nil |
| } |