cmd/coordinator: run benchmarks on try work
Benchmarks are treated as unit tests and distributed to the test
helpers, which allows them to fit in our 5m trybot budget.
Currently we only run the go1 and x/benchmarks. Running package
benchmarks is a TODO.
This feature is disabled by default, and is enabled by the
"farmer-run-bench" project attribute.
Updates golang/go#19178
Updates golang/go#19871
Change-Id: I9c3a14da60c3662e7e2cb4e71953060915cc4364
Reviewed-on: https://go-review.googlesource.com/38306
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/buildenv/envs.go b/buildenv/envs.go
index c376280..e647800 100644
--- a/buildenv/envs.go
+++ b/buildenv/envs.go
@@ -71,6 +71,9 @@
// DashURL is the base URL of the build dashboard, ending in a slash.
DashURL string
+ // PerfDataURL is the base URL of the benchmark storage server.
+ PerfDataURL string
+
// CoordinatorURL is the location from which the coordinator
// binary will be downloaded.
// This is only used by cmd/coordinator/buildongce/create.go when
@@ -171,6 +174,7 @@
KubeName: "buildlets",
KubeMachineType: "n1-standard-8",
DashURL: "https://go-dashboard-dev.appspot.com/",
+ PerfDataURL: "https://perfdata.golang.org",
CoordinatorURL: "https://storage.googleapis.com/dev-go-builder-data/coordinator",
CoordinatorName: "farmer",
BuildletBucket: "dev-go-builder-data",
@@ -192,6 +196,7 @@
KubeName: "buildlets",
KubeMachineType: "n1-standard-32",
DashURL: "https://build.golang.org/",
+ PerfDataURL: "https://perfdata.golang.org",
CoordinatorURL: "https://storage.googleapis.com/go-builder-data/coordinator",
CoordinatorName: "farmer",
BuildletBucket: "go-builder-data",
diff --git a/cmd/coordinator/Dockerfile.0 b/cmd/coordinator/Dockerfile.0
index c2ea01c..a51f7d4 100644
--- a/cmd/coordinator/Dockerfile.0
+++ b/cmd/coordinator/Dockerfile.0
@@ -12,6 +12,9 @@
RUN go get -d cloud.google.com/go/compute/metadata cloud.google.com/go/datastore cloud.google.com/go/storage
RUN cd /go/src/cloud.google.com/go && git reset --hard cd0da878c66091060d2e7403abd62192b3e387e0
+RUN go get -d golang.org/x/perf/storage
+RUN cd /go/src/golang.org/x/perf && git reset --hard b74b45749c47cd1edf5b64df78ecf13bd2dd944f
+
RUN go get -d golang.org/x/time/rate
RUN cd /go/src/golang.org/x/time/rate && git reset --hard f51c12702a4d776e4c1fa9b0fabab841babae631
diff --git a/cmd/coordinator/benchmarks.go b/cmd/coordinator/benchmarks.go
new file mode 100644
index 0000000..89388a2
--- /dev/null
+++ b/cmd/coordinator/benchmarks.go
@@ -0,0 +1,242 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "strings"
+ "time"
+
+ "golang.org/x/build/buildlet"
+)
+
+// benchRuns is the number of times to run each benchmark binary
+const benchRuns = 5
+
+type benchmarkItem struct {
+ binary string // name of binary relative to goroot
+ args []string // args to run binary with
+ preamble string // string to print before benchmark results (e.g. "pkg: test/bench/go1\n")
+ output []string // old, new benchmark output
+
+ build func(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) // how to build benchmark binary
+}
+
+func (b *benchmarkItem) name() string {
+ return b.binary + " " + strings.Join(b.args, " ")
+}
+
+// buildGo1 builds the Go 1 benchmarks.
+func (st *buildStatus) buildGo1(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) {
+ workDir, err := bc.WorkDir()
+ if err != nil {
+ return nil, err
+ }
+ var found bool
+ if err := bc.ListDir(path.Join(goroot, "test/bench/go1"), buildlet.ListDirOpts{}, func(e buildlet.DirEntry) {
+ switch e.Name() {
+ case "go1.test", "go1.test.exe":
+ found = true
+ }
+ }); err != nil {
+ return nil, err
+ }
+ if found {
+ return nil, nil
+ }
+ return bc.Exec(path.Join(goroot, "bin", "go"), buildlet.ExecOpts{
+ Output: w,
+ ExtraEnv: []string{"GOROOT=" + st.conf.FilePathJoin(workDir, goroot)},
+ Args: []string{"test", "-c"},
+ Dir: path.Join(goroot, "test/bench/go1"),
+ })
+}
+
+// buildXBenchmark builds a benchmark from x/benchmarks.
+func (st *buildStatus) buildXBenchmark(bc *buildlet.Client, goroot string, w io.Writer, rev, pkg, name string) (remoteErr, err error) {
+ workDir, err := bc.WorkDir()
+ if err != nil {
+ return nil, err
+ }
+ if err := bc.ListDir("gopath/src/golang.org/x/benchmarks", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil {
+ if err := st.fetchSubrepo(bc, "benchmarks", rev); err != nil {
+ return nil, err
+ }
+ }
+ return bc.Exec(path.Join(goroot, "bin/go"), buildlet.ExecOpts{
+ Output: w,
+ ExtraEnv: []string{
+ "GOROOT=" + st.conf.FilePathJoin(workDir, goroot),
+ "GOPATH=" + st.conf.FilePathJoin(workDir, "gopath"),
+ },
+ Args: []string{"build", "-o", st.conf.FilePathJoin(workDir, goroot, name), pkg},
+ })
+}
+
+func (st *buildStatus) enumerateBenchmarks(bc *buildlet.Client) ([]*benchmarkItem, error) {
+ workDir, err := bc.WorkDir()
+ if err != nil {
+ err = fmt.Errorf("buildBench, WorkDir: %v", err)
+ return nil, err
+ }
+ // Fetch x/benchmarks
+ rev := getRepoHead("benchmarks")
+ if rev == "" {
+ rev = "master" // should happen rarely; ok if it does.
+ }
+
+ if err := st.fetchSubrepo(bc, "benchmarks", rev); err != nil {
+ return nil, err
+ }
+
+ var out []*benchmarkItem
+
+ // These regexes shard the go1 tests so each shard takes about 20s, ensuring no test runs for
+ for _, re := range []string{`^Benchmark[BF]`, `^Benchmark[HR]`, `^Benchmark[^BFHR]`} {
+ out = append(out, &benchmarkItem{
+ binary: "test/bench/go1/go1.test",
+ args: []string{"-test.bench", re, "-test.benchmem"},
+ preamble: "pkg: test/bench/go1\n",
+ build: st.buildGo1,
+ })
+ }
+
+ // Enumerate x/benchmarks
+ var buf bytes.Buffer
+ remoteErr, err := bc.Exec("go/bin/go", buildlet.ExecOpts{
+ Output: &buf,
+ ExtraEnv: []string{
+ "GOROOT=" + st.conf.FilePathJoin(workDir, "go"),
+ "GOPATH=" + st.conf.FilePathJoin(workDir, "gopath"),
+ },
+ Args: []string{"list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, "golang.org/x/benchmarks/..."},
+ })
+ if remoteErr != nil {
+ return nil, remoteErr
+ }
+ if err != nil {
+ return nil, err
+ }
+ for _, pkg := range strings.Fields(buf.String()) {
+ pkg := pkg
+ name := "bench-" + path.Base(pkg) + ".exe"
+ out = append(out, &benchmarkItem{
+ binary: name, args: nil, build: func(bc *buildlet.Client, goroot string, w io.Writer) (error, error) {
+ return st.buildXBenchmark(bc, goroot, w, rev, pkg, name)
+ }})
+ }
+ // TODO(quentin): Enumerate package benchmarks that were affected by the CL
+ return out, nil
+}
+
+// runOneBenchBinary runs a binary on the buildlet and writes its output to w with a trailing newline.
+func (st *buildStatus) runOneBenchBinary(bc *buildlet.Client, w io.Writer, goroot string, path string, args []string) (remoteErr, err error) {
+ defer w.Write([]byte{'\n'})
+ workDir, err := bc.WorkDir()
+ if err != nil {
+ return nil, fmt.Errorf("runOneBenchBinary, WorkDir: %v", err)
+ }
+ // Some benchmarks need GOROOT so they can invoke cmd/go.
+ return bc.Exec(path, buildlet.ExecOpts{
+ Output: w,
+ Args: args,
+ Path: []string{"$WORKDIR/" + goroot + "/bin", "$PATH"},
+ ExtraEnv: []string{
+ "GOROOT=" + st.conf.FilePathJoin(workDir, goroot),
+ },
+ })
+}
+
+func (b *benchmarkItem) buildParent(st *buildStatus, bc *buildlet.Client, w io.Writer) error {
+ pbr := st.builderRev // copy
+ rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
+ if rev.Commit == nil {
+ return fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision)
+ }
+ if len(rev.Commit.Parents) == 0 {
+ // TODO(quentin): Log?
+ return errors.New("commit has no parent")
+ }
+ pbr.rev = rev.Commit.Parents[0].CommitID
+ if pbr.snapshotExists() {
+ return bc.PutTarFromURL(pbr.snapshotURL(), "go-parent")
+ }
+ if err := bc.PutTar(versionTgz(pbr.rev), "go-parent"); err != nil {
+ return err
+ }
+ srcTar, err := getSourceTgz(st, "go", pbr.rev)
+ if err != nil {
+ return err
+ }
+ if err := bc.PutTar(srcTar, "go-parent"); err != nil {
+ return err
+ }
+ remoteErr, err := st.runMake(bc, "go-parent", w)
+ if err != nil {
+ return err
+ }
+ return remoteErr
+}
+
+// run runs all the iterations of this benchmark on bc.
+// Build output is sent to w. Benchmark output is stored in b.output.
+// TODO(quentin): Take a list of commits so this can be used for non-try runs.
+func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
+ // Ensure we have a built parent repo.
+ if err := bc.ListDir("go-parent", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil {
+ sp := st.createSpan("bench_build_parent", bc.Name())
+ err := b.buildParent(st, bc, w)
+ sp.done(err)
+ if err != nil {
+ return nil, err
+ }
+ }
+ // Build benchmark.
+ for _, goroot := range []string{"go", "go-parent"} {
+ sp := st.createSpan("bench_build", fmt.Sprintf("%s/%s: %s", goroot, b.binary, bc.Name()))
+ remoteErr, err = b.build(bc, goroot, w)
+ sp.done(err)
+ if remoteErr != nil || err != nil {
+ return remoteErr, err
+ }
+ }
+
+ type commit struct {
+ path string
+ out bytes.Buffer
+ }
+ commits := []*commit{
+ {path: "go-parent"},
+ {path: "go"},
+ }
+
+ for _, c := range commits {
+ c.out.WriteString(b.preamble)
+ }
+
+ // Run bench binaries and capture the results
+ for i := 0; i < benchRuns; i++ {
+ for _, c := range commits {
+ fmt.Fprintf(&c.out, "iteration: %d\nstart-time: %s\n", i, time.Now().UTC().Format(time.RFC3339))
+ p := path.Join(c.path, b.binary)
+ sp := st.createSpan("run_one_bench", p)
+ remoteErr, err = st.runOneBenchBinary(bc, &c.out, c.path, p, b.args)
+ sp.done(err)
+ if err != nil || remoteErr != nil {
+ c.out.WriteTo(w)
+ return
+ }
+ }
+ }
+ b.output = []string{
+ commits[0].out.String(),
+ commits[1].out.String(),
+ }
+ return nil, nil
+}
diff --git a/cmd/coordinator/buildongce/create.go b/cmd/coordinator/buildongce/create.go
index 732bc80..d97ed4d 100644
--- a/cmd/coordinator/buildongce/create.go
+++ b/cmd/coordinator/buildongce/create.go
@@ -28,6 +28,7 @@
compute "google.golang.org/api/compute/v1"
dm "google.golang.org/api/deploymentmanager/v2"
"google.golang.org/api/googleapi"
+ oauth2api "google.golang.org/api/oauth2/v2"
)
var (
@@ -238,6 +239,7 @@
compute.ComputeScope,
compute.CloudPlatformScope,
datastore.ScopeDatastore,
+ oauth2api.UserinfoEmailScope,
},
},
},
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
index 9ad7799..acd2b47 100644
--- a/cmd/coordinator/coordinator.go
+++ b/cmd/coordinator/coordinator.go
@@ -19,7 +19,6 @@
"crypto/rand"
"crypto/sha1"
"crypto/tls"
- "encoding/json"
"errors"
"flag"
"fmt"
@@ -55,6 +54,7 @@
"golang.org/x/build/livelog"
"golang.org/x/build/types"
"golang.org/x/crypto/acme/autocert"
+ perfstorage "golang.org/x/perf/storage"
"golang.org/x/time/rate"
)
@@ -79,11 +79,17 @@
// finishes before destroying buildlets.
const devPause = false
+// stagingTryWork is a debug option to enable or disable running trybot work in staging.
+// If enabled, only open CLs containing "Run-StagingTryBot" in a comment will be run.
+const stagingTryWork = true
+
var (
- masterKeyFile = flag.String("masterkey", "", "Path to builder master key. Else fetched using GCE project attribute 'builder-master-key'.")
- mode = flag.String("mode", "", "Valid modes are 'dev', 'prod', or '' for auto-detect. dev means localhost development, not be confused with staging on go-dashboard-dev, which is still the 'prod' mode.")
- buildEnvName = flag.String("env", "", "The build environment configuration to use. Not required if running on GCE.")
- devEnableGCE = flag.Bool("dev_gce", false, "Whether or not to enable the GCE pool when in dev mode. The pool is enabled by default in prod mode.")
+ masterKeyFile = flag.String("masterkey", "", "Path to builder master key. Else fetched using GCE project attribute 'builder-master-key'.")
+ mode = flag.String("mode", "", "Valid modes are 'dev', 'prod', or '' for auto-detect. dev means localhost development, not be confused with staging on go-dashboard-dev, which is still the 'prod' mode.")
+ buildEnvName = flag.String("env", "", "The build environment configuration to use. Not required if running on GCE.")
+ devEnableGCE = flag.Bool("dev_gce", false, "Whether or not to enable the GCE pool when in dev mode. The pool is enabled by default in prod mode.")
+ shouldRunBench = flag.Bool("run_bench", false, "Whether or not to run benchmarks on trybot commits. Override by GCE project attribute 'farmer-run-bench'.")
+ perfServer = flag.String("perf_server", "", "Upload benchmark results to `server`. Overrides buildenv default for testing.")
)
// LOCK ORDER:
@@ -673,18 +679,9 @@
func findWork(work chan<- builderRev) error {
var bs types.BuildStatus
- res, err := http.Get(buildEnv.DashBase() + "?mode=json")
- if err != nil {
+ if err := dash("GET", "", url.Values{"mode": {"json"}}, nil, &bs); err != nil {
return err
}
- defer res.Body.Close()
- if err := json.NewDecoder(res.Body).Decode(&bs); err != nil {
- return err
- }
- if res.StatusCode != 200 {
- return fmt.Errorf("unexpected http status %v", res.Status)
- }
-
knownToDashboard := map[string]bool{} // keys are builder
for _, b := range bs.Builders {
knownToDashboard[b] = true
@@ -811,11 +808,15 @@
}
func findTryWork() error {
- if inStaging && true {
+ query := "label:Run-TryBot=1 label:TryBot-Result=0 status:open"
+ if inStaging && !stagingTryWork {
return nil
}
- cis, err := gerritClient.QueryChanges(context.Background(), "label:Run-TryBot=1 label:TryBot-Result=0 status:open", gerrit.QueryChangesOpt{
- Fields: []string{"CURRENT_REVISION"},
+ if inStaging {
+ query = `comment:"Run-StagingTryBot" label:TryBot-Result=0 status:open`
+ }
+ cis, err := gerritClient.QueryChanges(context.Background(), query, gerrit.QueryChangesOpt{
+ Fields: []string{"CURRENT_REVISION", "CURRENT_COMMIT"},
})
if err != nil {
return err
@@ -849,7 +850,7 @@
// already in progress
continue
}
- ts, err := newTrySet(key)
+ ts, err := newTrySet(key, ci)
if err != nil {
if err == errHeadUnknown {
continue // benign transient error
@@ -891,6 +892,7 @@
// immutable
tryKey
tryID string // "T" + 9 random hex
+ ci *gerrit.ChangeInfo
// mu guards state and errMsg
// See LOCK ORDER comment above.
@@ -900,16 +902,18 @@
}
type trySetState struct {
- remain int
- failed []string // build names
- builds []*buildStatus
+ remain int
+ failed []string // build names
+ builds []*buildStatus
+ benchResults []string // builder names
}
func (ts trySetState) clone() trySetState {
return trySetState{
- remain: ts.remain,
- failed: append([]string(nil), ts.failed...),
- builds: append([]*buildStatus(nil), ts.builds...),
+ remain: ts.remain,
+ failed: append([]string(nil), ts.failed...),
+ builds: append([]*buildStatus(nil), ts.builds...),
+ benchResults: append([]string(nil), ts.benchResults...),
}
}
@@ -917,11 +921,12 @@
// newTrySet creates a new trySet group of builders for a given key,
// the (Change-ID, Commit, Repo) tuple.
+// ci must contain the gerrit ChangeInfo for this change, and it must be fetched with the "CURRENT_REVISION" and "CURRENT_COMMIT" fields.
// It also starts goroutines for each build.
// This will fail if the current Go repo HEAD is unknown.
//
// Must hold statusMu.
-func newTrySet(key tryKey) (*trySet, error) {
+func newTrySet(key tryKey, ci *gerrit.ChangeInfo) (*trySet, error) {
goHead := getRepoHead("go")
if key.Repo != "go" && goHead == "" {
// We don't know the go HEAD yet (but we will)
@@ -939,6 +944,7 @@
ts := &trySet{
tryKey: key,
tryID: "T" + randHex(9),
+ ci: ci,
trySetState: trySetState{
remain: len(builders),
builds: make([]*buildStatus, len(builders)),
@@ -1078,15 +1084,20 @@
if !succeeded {
buildLog = bs.output.String()
}
+ hasBenchResults := bs.hasBenchResults
bs.mu.Unlock()
ts.mu.Lock()
+ if hasBenchResults {
+ ts.benchResults = append(ts.benchResults, bs.name)
+ }
ts.remain--
remain := ts.remain
if !succeeded {
ts.failed = append(ts.failed, bconf.Name)
}
numFail := len(ts.failed)
+ benchResults := append([]string(nil), ts.benchResults...)
ts.mu.Unlock()
if !succeeded {
@@ -1134,6 +1145,9 @@
score, msg = -1, fmt.Sprintf("%d of %d TryBots failed:\n%s\nConsult https://build.golang.org/ to see whether they are new failures.",
numFail, len(ts.builds), errMsg)
}
+ if len(benchResults) > 0 {
+ msg += fmt.Sprintf("\nBenchmark results for %s are available at:\nhttps://perf.golang.org/search?q=cl:%d+try:%s", strings.Join(benchResults, ", "), ts.ci.ChangeNumber, ts.tryID)
+ }
if err := gerritClient.SetReview(context.Background(), ts.ChangeTriple(), ts.Commit, gerrit.ReviewInput{
Message: msg,
Labels: map[string]int{
@@ -1624,6 +1638,14 @@
return rec
}
+// shouldBench returns whether we should attempt to run benchmarks
+func (st *buildStatus) shouldBench() bool {
+ if !*shouldRunBench {
+ return false
+ }
+ return st.isTry() && !st.isSubrepo() && st.conf.RunBench
+}
+
// runAllSharded runs make.bash and then shards the test execution.
// remoteErr and err are as described at the top of this file.
//
@@ -1633,12 +1655,14 @@
func (st *buildStatus) runAllSharded() (remoteErr, err error) {
st.getHelpersReadySoon()
- remoteErr, err = st.runMake()
- if err != nil {
- return nil, err
- }
- if remoteErr != nil {
- return fmt.Errorf("build failed: %v", remoteErr), nil
+ if !st.useSnapshot() {
+ remoteErr, err = st.runMake(st.bc, "go", st)
+ if err != nil {
+ return nil, err
+ }
+ if remoteErr != nil {
+ return fmt.Errorf("build failed: %v", remoteErr), nil
+ }
}
if st.conf.StopAfterMake {
return nil, nil
@@ -1653,6 +1677,7 @@
} else {
remoteErr, err = st.runTests(st.getHelpers())
}
+
if err != nil {
return nil, fmt.Errorf("runTests: %v", err)
}
@@ -1747,17 +1772,14 @@
}
// runMake builds the tool chain.
+// goroot is relative to the workdir with forward slashes.
+// w is the Writer to send build output to.
// remoteErr and err are as described at the top of this file.
-func (st *buildStatus) runMake() (remoteErr, err error) {
- // Don't do this if we're using a pre-built snapshot.
- if st.useSnapshot() {
- return nil, nil
- }
-
+func (st *buildStatus) runMake(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) {
// Build the source code.
makeSpan := st.createSpan("make", st.conf.MakeScript())
- remoteErr, err = st.bc.Exec(path.Join("go", st.conf.MakeScript()), buildlet.ExecOpts{
- Output: st,
+ remoteErr, err = bc.Exec(path.Join(goroot, st.conf.MakeScript()), buildlet.ExecOpts{
+ Output: w,
ExtraEnv: append(st.conf.Env(), "GOBIN="),
Debug: true,
Args: st.conf.MakeScriptArgs(),
@@ -1775,8 +1797,8 @@
// Need to run "go install -race std" before the snapshot + tests.
if st.conf.IsRace() {
sp := st.createSpan("install_race_std")
- remoteErr, err = st.bc.Exec("go/bin/go", buildlet.ExecOpts{
- Output: st,
+ remoteErr, err = bc.Exec(path.Join(goroot, "bin/go"), buildlet.ExecOpts{
+ Output: w,
ExtraEnv: append(st.conf.Env(), "GOBIN="),
Debug: true,
Args: []string{"install", "-race", "std"},
@@ -1867,7 +1889,7 @@
return sp.done(fmt.Errorf("writing VERSION tgz: %v", err))
}
- srcTar, err := getSourceTgz(st, "go", st.rev, st.isTry())
+ srcTar, err := getSourceTgz(st, "go", st.rev)
if err != nil {
return err
}
@@ -1998,7 +2020,7 @@
return false
}
-func (st *buildStatus) newTestSet(names []string) *testSet {
+func (st *buildStatus) newTestSet(names []string, benchmarks []*benchmarkItem) *testSet {
set := &testSet{
st: st,
}
@@ -2011,6 +2033,17 @@
done: make(chan token),
})
}
+ for _, bench := range benchmarks {
+ name := "bench:" + bench.name()
+ set.items = append(set.items, &testItem{
+ set: set,
+ name: name,
+ bench: bench,
+ duration: testDuration(name),
+ take: make(chan token, 1),
+ done: make(chan token),
+ })
+ }
return set
}
@@ -2296,9 +2329,21 @@
if secs, ok := fixedTestDuration[name]; ok {
return secs.Duration()
}
+ if strings.HasPrefix(name, "bench:") {
+ // Assume benchmarks are roughly 20 seconds per run.
+ return 2 * benchRuns * 20 * time.Second
+ }
return minGoTestSpeed * 2
}
+func (st *buildStatus) fetchSubrepo(bc *buildlet.Client, repo, rev string) error {
+ tgz, err := getSourceTgz(st, repo, rev)
+ if err != nil {
+ return err
+ }
+ return bc.PutTar(tgz, "gopath/src/"+subrepoPrefix+repo)
+}
+
func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
st.logEventTime("fetching_subrepo", st.subName)
@@ -2316,12 +2361,7 @@
// fetch checks out the provided sub-repo to the buildlet's workspace.
fetch := func(repo, rev string) error {
fetched[repo] = true
- isTry := st.trySet != nil && repo == st.subName // i.e. hit Gerrit directly
- tgz, err := getSourceTgz(st, repo, rev, isTry)
- if err != nil {
- return err
- }
- return st.bc.PutTar(tgz, "gopath/src/"+subrepoPrefix+repo)
+ return st.fetchSubrepo(st.bc, repo, rev)
}
// findDeps uses 'go list' on the checked out repo to find its
@@ -2415,7 +2455,16 @@
if err != nil {
return nil, fmt.Errorf("distTestList exec: %v", err)
}
- set := st.newTestSet(testNames)
+ var benches []*benchmarkItem
+ if st.shouldBench() {
+ sp := st.createSpan("enumerate_benchmarks")
+ b, err := st.enumerateBenchmarks(st.bc)
+ sp.done(err)
+ if err == nil {
+ benches = b
+ }
+ }
+ set := st.newTestSet(testNames, benches)
st.logEventTime("starting_tests", fmt.Sprintf("%d tests", len(set.items)))
startTime := time.Now()
@@ -2494,6 +2543,8 @@
close(buildletsGone)
}()
+ benchFiles := st.benchFiles()
+
var lastBanner string
var serialDuration time.Duration
for _, ti := range set.items {
@@ -2512,6 +2563,14 @@
}
}
+ if ti.bench != nil {
+ for i, s := range ti.bench.output {
+ if i < len(benchFiles) {
+ benchFiles[i].out.WriteString(s)
+ }
+ }
+ }
+
serialDuration += ti.execDuration
if len(ti.output) > 0 {
banner, out := parseOutputAndBanner(ti.output)
@@ -2542,9 +2601,70 @@
}
st.logEventTime("tests_complete", msg)
fmt.Fprintf(st, "\nAll tests passed.\n")
+ for _, f := range benchFiles {
+ if f.out.Len() > 0 {
+ st.hasBenchResults = true
+ }
+ }
+ if st.hasBenchResults {
+ sp := st.createSpan("upload_bench_results")
+ sp.done(st.uploadBenchResults(st.ctx, benchFiles))
+ }
return nil, nil
}
+func (st *buildStatus) uploadBenchResults(ctx context.Context, files []*benchFile) error {
+ s := *perfServer
+ if s == "" {
+ s = buildEnv.PerfDataURL
+ }
+ client := &perfstorage.Client{BaseURL: s, HTTPClient: oAuthHTTPClient}
+ u := client.NewUpload(ctx)
+ for _, b := range files {
+ w, err := u.CreateFile(b.name)
+ if err != nil {
+ u.Abort()
+ return err
+ }
+ if _, err := b.out.WriteTo(w); err != nil {
+ u.Abort()
+ return err
+ }
+ }
+ status, err := u.Commit()
+ if err != nil {
+ return err
+ }
+ st.logEventTime("bench_upload", status.UploadID)
+ return nil
+}
+
+type benchFile struct {
+ name string
+ out bytes.Buffer
+}
+
+func (st *buildStatus) benchFiles() []*benchFile {
+ if !st.shouldBench() {
+ return nil
+ }
+ rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
+ ps := rev.PatchSetNumber
+ benchFiles := []*benchFile{
+ {name: "orig.txt"},
+ {name: fmt.Sprintf("ps%d.txt", ps)},
+ }
+ fmt.Fprintf(&benchFiles[0].out, "cl: %d\nps: %d\ntry: %s\nbuildlet: %s\nbranch: %s\nrepo: https://go.googlesource.com/%s\n",
+ st.trySet.ci.ChangeNumber, ps, st.trySet.tryID,
+ st.name, st.trySet.ci.Branch, st.trySet.ci.Project,
+ )
+ if inStaging {
+ benchFiles[0].out.WriteString("staging: true\n")
+ }
+ benchFiles[1].out.Write(benchFiles[0].out.Bytes())
+ return benchFiles
+}
+
const (
banner = "XXXBANNERXXX:" // flag passed to dist
bannerPrefix = "\n" + banner // with the newline added by dist
@@ -2607,20 +2727,25 @@
var buf bytes.Buffer
t0 := time.Now()
timeout := execTimeout(names)
- remoteErr, err := bc.Exec(path.Join("go", "bin", "go"), buildlet.ExecOpts{
- // We set Dir to "." instead of the default ("go/bin") so when the dist tests
- // try to run os/exec.Command("go", "test", ...), the LookPath of "go" doesn't
- // return "./go.exe" (which exists in the current directory: "go/bin") and then
- // fail when dist tries to run the binary in dir "$GOROOT/src", since
- // "$GOROOT/src" + "./go.exe" doesn't exist. Perhaps LookPath should return
- // an absolute path.
- Dir: ".",
- Output: &buf, // see "maybe stream lines" TODO below
- ExtraEnv: append(st.conf.Env(), "GOROOT="+goroot),
- Timeout: timeout,
- Path: []string{"$WORKDIR/go/bin", "$PATH"},
- Args: args,
- })
+ var remoteErr, err error
+ if ti := tis[0]; ti.bench != nil {
+ remoteErr, err = ti.bench.run(st, bc, &buf)
+ } else {
+ remoteErr, err = bc.Exec(path.Join("go", "bin", "go"), buildlet.ExecOpts{
+ // We set Dir to "." instead of the default ("go/bin") so when the dist tests
+ // try to run os/exec.Command("go", "test", ...), the LookPath of "go" doesn't
+ // return "./go.exe" (which exists in the current directory: "go/bin") and then
+ // fail when dist tries to run the binary in dir "$GOROOT/src", since
+ // "$GOROOT/src" + "./go.exe" doesn't exist. Perhaps LookPath should return
+ // an absolute path.
+ Dir: ".",
+ Output: &buf, // see "maybe stream lines" TODO below
+ ExtraEnv: append(st.conf.Env(), "GOROOT="+goroot),
+ Timeout: timeout,
+ Path: []string{"$WORKDIR/go/bin", "$PATH"},
+ Args: args,
+ })
+ }
execDuration := time.Since(t0)
sp.done(err)
if err != nil {
@@ -2751,6 +2876,8 @@
name string // "go_test:sort"
duration time.Duration // optional approximate size
+ bench *benchmarkItem // If populated, this is a benchmark instead of a regular test.
+
take chan token // buffered size 1: sending takes ownership of rest of fields:
done chan token // closed when done; guards output & failed
@@ -2829,6 +2956,8 @@
hasBuildlet int32 // atomic: non-zero if this build has a buildlet; for status.go.
+ hasBenchResults bool // set by runTests, may only be used when build() returns.
+
mu sync.Mutex // guards following
failURL string // if non-empty, permanent URL of failure
bc *buildlet.Client // nil initially, until pool returns one
@@ -3093,7 +3222,7 @@
// repo is go.googlesource.com repo ("go", "net", etc)
// rev is git revision.
-func getSourceTgz(sl spanLogger, repo, rev string, isTry bool) (tgz io.Reader, err error) {
+func getSourceTgz(sl spanLogger, repo, rev string) (tgz io.Reader, err error) {
sp := sl.createSpan("get_source")
defer func() { sp.done(err) }()
diff --git a/cmd/coordinator/dash.go b/cmd/coordinator/dash.go
index 01c82cc..3c7afd4 100644
--- a/cmd/coordinator/dash.go
+++ b/cmd/coordinator/dash.go
@@ -79,21 +79,10 @@
// Read JSON-encoded Response into provided resp
// and return an error if present.
- var result = struct {
- Response interface{}
- Error string
- }{
- // Put the provided resp in here as it can be a pointer to
- // some value we should unmarshal into.
- Response: resp,
- }
- if err = json.Unmarshal(body.Bytes(), &result); err != nil {
+ if err = json.Unmarshal(body.Bytes(), resp); err != nil {
log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
return err
}
- if result.Error != "" {
- return errors.New(result.Error)
- }
return nil
}
@@ -121,7 +110,17 @@
log.Printf("In dev mode, not recording result: %v", req)
return nil
}
- return dash("POST", "result", args, req, nil)
+ var result struct {
+ Response interface{}
+ Error string
+ }
+ if err := dash("POST", "result", args, req, &result); err != nil {
+ return err
+ }
+ if result.Error != "" {
+ return errors.New(result.Error)
+ }
+ return nil
}
// pingDashboard runs in its own goroutine, created periodically to
diff --git a/cmd/coordinator/gce.go b/cmd/coordinator/gce.go
index aec1f9a..29c53a6 100644
--- a/cmd/coordinator/gce.go
+++ b/cmd/coordinator/gce.go
@@ -15,6 +15,7 @@
"io"
"io/ioutil"
"log"
+ "net/http"
"os"
"path"
"sort"
@@ -36,6 +37,7 @@
"golang.org/x/oauth2/google"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
+ oauth2api "google.golang.org/api/oauth2/v2"
)
func init() {
@@ -66,6 +68,10 @@
initGCECalled bool
)
+// oAuthHTTPClient is the OAuth2 HTTP client used to make API calls to Google Cloud APIs.
+// It is initialized by initGCE.
+var oAuthHTTPClient *http.Client
+
func initGCE() error {
initGCECalled = true
var err error
@@ -106,6 +112,10 @@
return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.")
}
+
+ if value, err := metadata.ProjectAttributeValue("farmer-run-bench"); err == nil {
+ *shouldRunBench, _ = strconv.ParseBool(value)
+ }
}
cfgDump, _ := json.MarshalIndent(buildEnv, "", " ")
@@ -128,7 +138,7 @@
}
}
- tokenSource, err = google.DefaultTokenSource(ctx, compute.CloudPlatformScope)
+ tokenSource, err = google.DefaultTokenSource(ctx, oauth2api.UserinfoEmailScope, compute.CloudPlatformScope)
if err != nil {
if *mode == "dev" {
// don't try to do anything else with GCE, as it will likely fail
@@ -136,8 +146,8 @@
}
log.Fatalf("failed to get a token source: %v", err)
}
- httpClient := oauth2.NewClient(ctx, tokenSource)
- computeService, _ = compute.New(httpClient)
+ oAuthHTTPClient = oauth2.NewClient(ctx, tokenSource)
+ computeService, _ = compute.New(oAuthHTTPClient)
errTryDeps = checkTryBuildDeps()
if errTryDeps != nil {
log.Printf("TryBot builders disabled due to error: %v", errTryDeps)
diff --git a/dashboard/builders.go b/dashboard/builders.go
index 0689024..2376f2e 100644
--- a/dashboard/builders.go
+++ b/dashboard/builders.go
@@ -431,6 +431,9 @@
// the tarball in under ~5 minutes.
SkipSnapshot bool
+ // RunBench causes the coordinator to run benchmarks on this buildlet type.
+ RunBench bool
+
// StopAfterMake causes the build to stop after the make
// script completes, returning its result as the result of the
// whole build. It does not run or compile any of the tests,
@@ -712,6 +715,7 @@
TryBot: true,
numTestHelpers: 2,
numTryTestHelpers: 4,
+ RunBench: true,
})
addBuilder(BuildConfig{
Name: "freebsd-amd64-110",
@@ -738,6 +742,7 @@
env: []string{"GOARCH=386", "GOHOSTARCH=386"},
numTestHelpers: 1,
numTryTestHelpers: 3,
+ RunBench: true,
})
addBuilder(BuildConfig{
Name: "linux-386-387",
@@ -750,6 +755,7 @@
HostType: "host-linux-kubestd",
TryBot: true,
numTestHelpers: 3,
+ RunBench: true,
})
addBuilder(BuildConfig{
Name: "linux-amd64-alpine",
@@ -849,6 +855,7 @@
FlakyNet: true,
numTestHelpers: 2,
numTryTestHelpers: 7,
+ RunBench: true,
})
addBuilder(BuildConfig{
Name: "linux-arm-nativemake",
@@ -917,6 +924,7 @@
TryBot: true,
numTestHelpers: 1,
numTryTestHelpers: 5,
+ RunBench: true,
})
addBuilder(BuildConfig{
Name: "windows-amd64-race",
