cmd/builder/bench.go - build - Git at Google

 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package main

 import (
 	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 )

 // benchHash benchmarks a single commit.
 func (b *Builder) benchHash(hash string, benchs []string) error {
 	if *verbose {
 		log.Println(b.name, "benchmarking", hash)
 	}

 	res := &PerfResult{Hash: hash, Benchmark: "meta-done"}

 	// Create place in which to do work.
 	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
 	// Prepare a workpath if we don't have one we can reuse.
 	update := false
 	if b.lastWorkpath != workpath {
 		if err := os.Mkdir(workpath, mkdirPerm); err != nil {
 			return err
 		}
 		buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
 		if err != nil {
 			removePath(workpath)
 			// record failure
 			res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
 			return b.recordPerfResult(res)
 		}
 		b.lastWorkpath = workpath
 		update = true
 	}

 	// Build the benchmark binary.
 	benchBin, buildLog, err := b.buildBenchmark(workpath, update)
 	if err != nil {
 		// record failure
 		res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
 		return b.recordPerfResult(res)
 	}

 	benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
 	if benchmark != "" {
 		res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
 		res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
 		if err = b.recordPerfResult(res); err != nil {
 			return fmt.Errorf("recordResult: %s", err)
 		}
 	}

 	if last {
 		// All benchmarks have beed executed, don't need workpath anymore.
 		removePath(b.lastWorkpath)
 		b.lastWorkpath = ""
 		// Notify the app.
 		res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
 		if err = b.recordPerfResult(res); err != nil {
 			return fmt.Errorf("recordResult: %s", err)
 		}
 	}

 	return nil
 }

 // buildBenchmark builds the benchmark binary.
 func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
 	goroot := filepath.Join(workpath, "go")
 	gobin := filepath.Join(goroot, "bin", "go") + exeExt
 	gopath := filepath.Join(*buildroot, "gopath")
 	env := append([]string{
 		"GOROOT=" + goroot,
 		"GOPATH=" + gopath},
 		b.envv()...)
 	// First, download without installing.
 	args := []string{"get", "-d"}
 	if update {
 		args = append(args, "-u")
 	}
 	args = append(args, *benchPath)
 	var buildlog bytes.Buffer
 	runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
 	err = run(exec.Command(gobin, args...), runOpts...)
 	if err != nil {
 		fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
 		return "", buildlog.String(), err
 	}
 	// Then, build into workpath.
 	benchBin = filepath.Join(workpath, "benchbin") + exeExt
 	args = []string{"build", "-o", benchBin, *benchPath}
 	buildlog.Reset()
 	err = run(exec.Command(gobin, args...), runOpts...)
 	if err != nil {
 		fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
 		return "", buildlog.String(), err
 	}
 	return benchBin, "", nil
 }

 // chooseBenchmark chooses the next benchmark to run
 // based on the list of available benchmarks, already executed benchmarks
 // and -benchcpu list.
 func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
 	var out bytes.Buffer
 	err := run(exec.Command(benchBin), allOutput(&out))
 	if err != nil {
 		log.Printf("Failed to query benchmark list: %v\n%s", err, &out)
 		last = true
 		return
 	}
 	outStr := out.String()
 	nlIdx := strings.Index(outStr, "\n")
 	if nlIdx < 0 {
 		log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
 		last = true
 		return
 	}
 	localBenchs := strings.Split(outStr[:nlIdx], ",")
 	benchsMap := make(map[string]bool)
 	for _, b := range doneBenchs {
 		benchsMap[b] = true
 	}
 	cnt := 0
 	// We want to run all benchmarks with GOMAXPROCS=1 first.
 	for i, procs1 := range benchCPU {
 		for _, bench1 := range localBenchs {
 			if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
 				continue
 			}
 			cnt++
 			if cnt == 1 {
 				bench = bench1
 				procs = procs1
 				if i < len(benchAffinity) {
 					affinity = benchAffinity[i]
 				}
 			}
 		}
 	}
 	last = cnt <= 1
 	return
 }

 // executeBenchmark runs a single benchmark and parses its output.
 func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
 	// Benchmarks runs mutually exclusive with other activities.
 	benchMutex.RUnlock()
 	defer benchMutex.RLock()
 	benchMutex.Lock()
 	defer benchMutex.Unlock()

 	log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)

 	// The benchmark executes 'go build'/'go tool',
 	// so we need properly setup env.
 	env := append([]string{
 		"GOROOT=" + filepath.Join(workpath, "go"),
 		"PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
 		"GODEBUG=gctrace=1", // since Go1.2
 		"GOGCTRACE=1",       // before Go1.2
 		fmt.Sprintf("GOMAXPROCS=%v", procs)},
 		b.envv()...)
 	args := []string{
 		"-bench", bench,
 		"-benchmem", strconv.Itoa(*benchMem),
 		"-benchtime", benchTime.String(),
 		"-benchnum", strconv.Itoa(*benchNum),
 		"-tmpdir", workpath}
 	if affinity != 0 {
 		args = append(args, "-affinity", strconv.Itoa(affinity))
 	}
 	benchlog := new(bytes.Buffer)
 	err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
 	if strip := benchlog.Len() - 512<<10; strip > 0 {
 		// Leave the last 512K, that part contains metrics.
 		benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
 	}
 	artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
 	if err != nil {
 		if err != nil {
 			log.Printf("Failed to execute benchmark '%v': %v", bench, err)
 			ok = false
 		}
 		return
 	}

 	metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
 	if err != nil {
 		log.Printf("Failed to parse benchmark output: %v", err)
 		ok = false
 		return
 	}
 	metrics = metrics1
 	artifacts = append(artifacts, artifacts1...)
 	ok = true
 	return
 }

 // parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
 func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
 	s := bufio.NewScanner(out)
 	metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
 	fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
 	for s.Scan() {
 		ln := s.Text()
 		if ss := metricRe.FindStringSubmatch(ln); ss != nil {
 			var v uint64
 			v, err = strconv.ParseUint(ss[2], 10, 64)
 			if err != nil {
 				err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
 				return
 			}
 			metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
 		} else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
 			var buf []byte
 			buf, err = ioutil.ReadFile(ss[2])
 			if err != nil {
 				err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
 				return
 			}
 			artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
 		}
 	}
 	return
 }

 // needsBenchmarking determines whether the commit needs benchmarking.
 func needsBenchmarking(log *HgLog) bool {
 	// Do not benchmark branch commits, they are usually not interesting
 	// and fall out of the trunk succession.
 	if log.Branch != "" {
 		return false
 	}
 	// Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
 	for _, f := range strings.Split(log.Files, " ") {
 		if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
 			!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
 			return true
 		}
 	}
 	return false
 }
	// Copyright 2013 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package main

	import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	)

	// benchHash benchmarks a single commit.
	func (b *Builder) benchHash(hash string, benchs []string) error {
	if *verbose {
	log.Println(b.name, "benchmarking", hash)
	}

	res := &PerfResult{Hash: hash, Benchmark: "meta-done"}

	// Create place in which to do work.
	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
	// Prepare a workpath if we don't have one we can reuse.
	update := false
	if b.lastWorkpath != workpath {
	if err := os.Mkdir(workpath, mkdirPerm); err != nil {
	return err
	}
	buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
	if err != nil {
	removePath(workpath)
	// record failure
	res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
	return b.recordPerfResult(res)
	}
	b.lastWorkpath = workpath
	update = true
	}

	// Build the benchmark binary.
	benchBin, buildLog, err := b.buildBenchmark(workpath, update)
	if err != nil {
	// record failure
	res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
	return b.recordPerfResult(res)
	}

	benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
	if benchmark != "" {
	res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
	res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
	if err = b.recordPerfResult(res); err != nil {
	return fmt.Errorf("recordResult: %s", err)
	}
	}

	if last {
	// All benchmarks have beed executed, don't need workpath anymore.
	removePath(b.lastWorkpath)
	b.lastWorkpath = ""
	// Notify the app.
	res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
	if err = b.recordPerfResult(res); err != nil {
	return fmt.Errorf("recordResult: %s", err)
	}
	}

	return nil
	}

	// buildBenchmark builds the benchmark binary.
	func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
	goroot := filepath.Join(workpath, "go")
	gobin := filepath.Join(goroot, "bin", "go") + exeExt
	gopath := filepath.Join(*buildroot, "gopath")
	env := append([]string{
	"GOROOT=" + goroot,
	"GOPATH=" + gopath},
	b.envv()...)
	// First, download without installing.
	args := []string{"get", "-d"}
	if update {
	args = append(args, "-u")
	}
	args = append(args, *benchPath)
	var buildlog bytes.Buffer
	runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
	fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
	return "", buildlog.String(), err
	}
	// Then, build into workpath.
	benchBin = filepath.Join(workpath, "benchbin") + exeExt
	args = []string{"build", "-o", benchBin, *benchPath}
	buildlog.Reset()
	err = run(exec.Command(gobin, args...), runOpts...)
	if err != nil {
	fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
	return "", buildlog.String(), err
	}
	return benchBin, "", nil
	}

	// chooseBenchmark chooses the next benchmark to run
	// based on the list of available benchmarks, already executed benchmarks
	// and -benchcpu list.
	func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
	var out bytes.Buffer
	err := run(exec.Command(benchBin), allOutput(&out))
	if err != nil {
	log.Printf("Failed to query benchmark list: %v\n%s", err, &out)
	last = true
	return
	}
	outStr := out.String()
	nlIdx := strings.Index(outStr, "\n")
	if nlIdx < 0 {
	log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
	last = true
	return
	}
	localBenchs := strings.Split(outStr[:nlIdx], ",")
	benchsMap := make(map[string]bool)
	for _, b := range doneBenchs {
	benchsMap[b] = true
	}
	cnt := 0
	// We want to run all benchmarks with GOMAXPROCS=1 first.
	for i, procs1 := range benchCPU {
	for _, bench1 := range localBenchs {
	if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
	continue
	}
	cnt++
	if cnt == 1 {
	bench = bench1
	procs = procs1
	if i < len(benchAffinity) {
	affinity = benchAffinity[i]
	}
	}
	}
	}
	last = cnt <= 1
	return
	}

	// executeBenchmark runs a single benchmark and parses its output.
	func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
	// Benchmarks runs mutually exclusive with other activities.
	benchMutex.RUnlock()
	defer benchMutex.RLock()
	benchMutex.Lock()
	defer benchMutex.Unlock()

	log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)

	// The benchmark executes 'go build'/'go tool',
	// so we need properly setup env.
	env := append([]string{
	"GOROOT=" + filepath.Join(workpath, "go"),
	"PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
	"GODEBUG=gctrace=1", // since Go1.2
	"GOGCTRACE=1", // before Go1.2
	fmt.Sprintf("GOMAXPROCS=%v", procs)},
	b.envv()...)
	args := []string{
	"-bench", bench,
	"-benchmem", strconv.Itoa(*benchMem),
	"-benchtime", benchTime.String(),
	"-benchnum", strconv.Itoa(*benchNum),
	"-tmpdir", workpath}
	if affinity != 0 {
	args = append(args, "-affinity", strconv.Itoa(affinity))
	}
	benchlog := new(bytes.Buffer)
	err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
	if strip := benchlog.Len() - 512<<10; strip > 0 {
	// Leave the last 512K, that part contains metrics.
	benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
	}
	artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
	if err != nil {
	if err != nil {
	log.Printf("Failed to execute benchmark '%v': %v", bench, err)
	ok = false
	}
	return
	}

	metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
	if err != nil {
	log.Printf("Failed to parse benchmark output: %v", err)
	ok = false
	return
	}
	metrics = metrics1
	artifacts = append(artifacts, artifacts1...)
	ok = true
	return
	}

	// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
	func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
	s := bufio.NewScanner(out)
	metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
	fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
	for s.Scan() {
	ln := s.Text()
	if ss := metricRe.FindStringSubmatch(ln); ss != nil {
	var v uint64
	v, err = strconv.ParseUint(ss[2], 10, 64)
	if err != nil {
	err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
	return
	}
	metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
	} else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
	var buf []byte
	buf, err = ioutil.ReadFile(ss[2])
	if err != nil {
	err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
	return
	}
	artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
	}
	}
	return
	}

	// needsBenchmarking determines whether the commit needs benchmarking.
	func needsBenchmarking(log *HgLog) bool {
	// Do not benchmark branch commits, they are usually not interesting
	// and fall out of the trunk succession.
	if log.Branch != "" {
	return false
	}
	// Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
	for _, f := range strings.Split(log.Files, " ") {
	if (strings.HasPrefix(f, "include") \|\| strings.HasPrefix(f, "src")) &&
	!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
	return true
	}
	}
	return false
	}