blob: 8a0e07c7eff600aadaeec948115b6304f41f9508 [file] [log] [blame]
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"golang.org/x/benchmarks/sweet/common"
"golang.org/x/benchmarks/sweet/common/diagnostics"
"golang.org/x/benchmarks/sweet/common/fileutil"
"golang.org/x/benchmarks/sweet/common/log"
"golang.org/x/benchmarks/sweet/generators"
"golang.org/x/benchmarks/sweet/harnesses"
)
var allBenchmarks = []benchmark{
{
name: "biogo-igor",
description: "Reports feature family groupings in pairwise alignment data",
harness: harnesses.BiogoIgor(),
generator: generators.BiogoIgor(),
},
{
name: "biogo-krishna",
description: "Performs pairwise alignment of a target sequence against itself",
harness: harnesses.BiogoKrishna(),
generator: generators.BiogoKrishna(),
},
{
name: "bleve-index",
description: "Indexes a subset of Wikipedia into a search index",
harness: harnesses.BleveIndex(),
generator: generators.BleveIndex(),
},
{
name: "cockroachdb",
description: "Distributed database",
harness: harnesses.CockroachDB{},
generator: generators.None{},
},
{
name: "etcd",
description: "Distributed key-value store",
harness: harnesses.Etcd{},
generator: generators.None{},
},
{
name: "go-build",
description: "Go build command",
harness: harnesses.GoBuild{},
generator: generators.None{},
},
{
name: "gopher-lua",
description: "Runs a k-nucleotide benchmark written in Lua on a Go-based Lua VM",
harness: harnesses.GopherLua(),
generator: generators.GopherLua(),
},
{
name: "gvisor",
description: "Container runtime sandbox for Linux (requires root)",
harness: harnesses.GVisor{},
generator: generators.GVisor{},
},
{
name: "markdown",
description: "Renders a corpus of markdown documents to XHTML",
harness: harnesses.Markdown(),
generator: generators.Markdown(),
},
{
name: "tile38",
description: "Redis-like geospatial database and geofencing server",
harness: harnesses.Tile38{},
generator: generators.Tile38{},
},
}
var allBenchmarksMap = func() map[string]*benchmark {
m := make(map[string]*benchmark)
for i := range allBenchmarks {
m[allBenchmarks[i].name] = &allBenchmarks[i]
}
return m
}()
var benchmarkGroups = func() map[string][]*benchmark {
m := make(map[string][]*benchmark)
m["default"] = []*benchmark{
allBenchmarksMap["biogo-igor"],
allBenchmarksMap["biogo-krishna"],
allBenchmarksMap["bleve-index"],
allBenchmarksMap["cockroachdb"],
allBenchmarksMap["etcd"],
allBenchmarksMap["go-build"],
allBenchmarksMap["gopher-lua"],
}
if runtime.GOARCH == "amd64" {
m["default"] = append(m["default"], allBenchmarksMap["gvisor"])
}
m["default"] = append(m["default"],
allBenchmarksMap["markdown"],
allBenchmarksMap["tile38"],
)
for i := range allBenchmarks {
m["all"] = append(m["all"], &allBenchmarks[i])
}
return m
}()
func benchmarkNames(b []*benchmark) (s []string) {
for _, bench := range b {
s = append(s, bench.name)
}
return
}
func mkdirAll(path string) error {
log.CommandPrintf("mkdir -p %s", path)
return os.MkdirAll(path, os.ModePerm)
}
func copyDirContents(dst, src string) error {
log.CommandPrintf("cp -r %s/* %s", src, dst)
return fileutil.CopyDir(dst, src, nil)
}
func rmDirContents(dir string) error {
log.CommandPrintf("rm -rf %s/*", dir)
fs, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, fi := range fs {
if err := os.RemoveAll(filepath.Join(dir, fi.Name())); err != nil {
return err
}
}
return nil
}
func readFileTail(f *os.File) (data string, err error) {
const maxLines = 20
const maxBytes = 16 << 10
const block = 4096
// Read the file backwards.
stat, err := f.Stat()
if err != nil {
return "", err
}
pos := stat.Size()
nLines, nBytes := -1, 0
blocks := make([][]byte, 0, 10)
for pos > 0 {
// Back up by a block
want := int64(block)
if pos < block {
want = pos
}
pos -= want
// Read
buf := make([]byte, want)
n, readErr := f.ReadAt(buf, pos)
buf = buf[:n]
// Apply byte limit
if nBytes+n > maxBytes {
n = maxBytes - nBytes
buf = buf[len(buf)-n:]
}
nBytes += n
// Apply line limit
nl := len(buf)
for {
nl = bytes.LastIndexByte(buf[:nl], '\n')
if nl == -1 {
break
}
nLines++
if nLines == maxLines {
buf = buf[nl+1:]
break
}
}
blocks = append(blocks, buf)
if nLines >= maxLines || nBytes >= maxBytes {
break
}
if readErr != nil {
return "", err
}
}
// Build final result
var str strings.Builder
for i := len(blocks) - 1; i >= 0; i-- {
str.Write(blocks[i])
}
return str.String(), nil
}
type benchmark struct {
name string
description string
harness common.Harness
generator common.Generator
}
func (b *benchmark) execute(cfgs []*common.Config, r *runCfg) error {
log.Printf("Setting up benchmark: %s", b.name)
// Compute top-level directories for this benchmark to work in.
benchDir := filepath.Join(r.benchDir, b.name)
topDir := filepath.Join(r.workDir, b.name)
srcDir := filepath.Join(topDir, "src")
// Check if assets for this benchmark exist. Not all benchmarks have assets!
var hasAssets bool
assetsFSDir := b.name
if f, err := r.assetsFS.Open(assetsFSDir); err == nil {
fi, err := f.Stat()
if err != nil {
f.Close()
return err
}
if !fi.IsDir() {
f.Close()
return fmt.Errorf("found assets file for %s instead of directory", b.name)
}
f.Close()
hasAssets = true
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
// Retrieve the benchmark's source, if needed. If execute is called
// multiple times, this will already be done.
_, err := os.Stat(srcDir)
if os.IsNotExist(err) {
gcfg := &common.GetConfig{
SrcDir: srcDir,
Short: r.short,
}
if err := b.harness.Get(gcfg); err != nil {
return fmt.Errorf("retrieving source for %s: %v", b.name, err)
}
}
// Create the results directory for the benchmark.
resultsDir := r.benchmarkResultsDir(b)
if err := mkdirAll(resultsDir); err != nil {
return fmt.Errorf("creating results directory for %s: %v", b.name, err)
}
// Perform a setup step for each config for the benchmark.
setups := make([]common.RunConfig, 0, len(cfgs))
for _, pcfg := range cfgs {
// Local copy for per-benchmark environment adjustments.
cfg := pcfg.Copy()
// Create directory hierarchy for benchmarks.
workDir := filepath.Join(topDir, cfg.Name)
binDir := filepath.Join(workDir, "bin")
tmpDir := filepath.Join(workDir, "tmp")
assetsDir := filepath.Join(workDir, "assets")
if err := mkdirAll(binDir); err != nil {
return fmt.Errorf("create %s bin for %s: %v", b.name, cfg.Name, err)
}
if err := mkdirAll(srcDir); err != nil {
return fmt.Errorf("create %s src for %s: %v", b.name, cfg.Name, err)
}
if err := mkdirAll(tmpDir); err != nil {
return fmt.Errorf("create %s tmp for %s: %v", b.name, cfg.Name, err)
}
if hasAssets {
if err := mkdirAll(assetsDir); err != nil {
return fmt.Errorf("create %s assets dir for %s: %v", b.name, cfg.Name, err)
}
}
// Add PGO if profile specified for this benchmark, otherwise
// explicitly disable it to avoid default.pgo files.
pgo, ok := cfg.PGOFiles[b.name]
if !ok {
pgo = "off"
}
goflags, ok := cfg.BuildEnv.Lookup("GOFLAGS")
if ok {
goflags += " "
}
goflags += fmt.Sprintf("-pgo=%s", pgo)
cfg.BuildEnv.Env = cfg.BuildEnv.MustSet("GOFLAGS=" + goflags)
// Build the benchmark (application and any other necessary components).
bcfg := common.BuildConfig{
BinDir: binDir,
SrcDir: srcDir,
BenchDir: benchDir,
Short: r.short,
}
if err := b.harness.Build(cfg, &bcfg); err != nil {
return fmt.Errorf("build %s for %s: %v", b.name, cfg.Name, err)
}
// Generate any args to funnel through to benchmarks.
args := []string{}
if r.dumpCore {
// Create a directory for the core files to live in.
resultsCoresDir := filepath.Join(resultsDir, "core")
mkdirAll(resultsCoresDir)
// We need to pass an argument to the benchmark binary to generate
// a core file. See benchmarks/internal/driver for details.
args = append(args, "-dump-cores", resultsCoresDir)
// Copy the bin directory so that the binaries may be used to analyze
// the core dump.
resultsBinDir := filepath.Join(resultsDir, "bin")
mkdirAll(resultsBinDir)
copyDirContents(resultsBinDir, binDir)
}
if !cfg.Diagnostics.Empty() {
// Create a directory for any profile files to live in.
resultsProfilesDir := r.runProfilesDir(b, cfg)
mkdirAll(resultsProfilesDir)
// We need to pass arguments to the benchmark binary to generate
// profiles. See benchmarks/internal/driver for details.
dc := diagnostics.DriverConfig{ResultsDir: resultsProfilesDir, ConfigSet: cfg.Diagnostics}
args = append(args, dc.DriverArgs()...)
}
// Create log and results file.
results, err := os.Create(filepath.Join(resultsDir, fmt.Sprintf("%s.results", cfg.Name)))
if err != nil {
return fmt.Errorf("create %s results file for %s: %v", b.name, cfg.Name, err)
}
defer results.Close()
log, err := os.Create(filepath.Join(resultsDir, fmt.Sprintf("%s.log", cfg.Name)))
if err != nil {
return fmt.Errorf("create %s log file for %s: %v", b.name, cfg.Name, err)
}
defer results.Close()
setups = append(setups, common.RunConfig{
BinDir: binDir,
TmpDir: tmpDir,
AssetsDir: assetsDir,
Args: args,
Results: results,
Log: log,
Short: r.short,
})
}
for j := 0; j < r.count; j++ {
// Execute the benchmark for each configuration.
for i, setup := range setups {
if hasAssets {
// Set up assets directory for test run.
r.logCopyDirCommand(b.name, setup.AssetsDir)
if err := fileutil.CopyDir(setup.AssetsDir, assetsFSDir, r.assetsFS); err != nil {
return err
}
}
log.Printf("Running benchmark %s for %s: run %d", b.name, cfgs[i].Name, j+1)
// Force a GC now because we're about to turn it off.
runtime.GC()
// Hold your breath: we're turning off GC for the duration of the
// run so that the suite's GC doesn't start blasting on all Ps,
// introducing undue noise into the experiments.
gogc := debug.SetGCPercent(-1)
if err := b.harness.Run(cfgs[i], &setup); err != nil {
debug.SetGCPercent(gogc)
// Useful error messages are often in the log. Grab the end.
logTail, tailErr := readFileTail(setup.Log)
if tailErr != nil {
logTail = fmt.Sprintf("error reading log tail: %s", tailErr)
}
setup.Log.Close()
return fmt.Errorf("run benchmark %s for config %s: %v\nLog tail:\n%s", b.name, cfgs[i].Name, err, logTail)
}
debug.SetGCPercent(gogc)
// Clean up tmp directory so benchmarks may assume it's empty.
if err := rmDirContents(setup.TmpDir); err != nil {
return err
}
if hasAssets {
// Clean up assets directory just in case any of the files were written to.
if err := rmDirContents(setup.AssetsDir); err != nil {
return err
}
}
}
}
return nil
}