cmd/coordinator: use historical test timing data for critical path scheduling
We previously hard-coded a Linux-only static map of 2015 data in for
the critical path scheduling of cmd/dist tests over N sharded
buildlets. That worked well only for Linux and only in 2015.
Instead, query BigQuery to find out what the recent timing data looks
like for all builders.
I'd started to work on this back in CL 30716 (Oct 2016) but apparently
never finished. Yay me. But skip the writing-to-CSV step. BigQuery is
much faster than I remember (maybe it got faster?), so just query it
directly. The query takes about 2 seconds, and we only do it every
hour (which is still overkill; daily is probably fine).
Change-Id: I498fc09dfaf24fb1f11b2c0ab4b952b2f15f9c32
Reviewed-on: https://go-review.googlesource.com/c/160037
Reviewed-by: Andrew Bonventre <andybons@golang.org>
diff --git a/cmd/buildstats/buildstats.go b/cmd/buildstats/buildstats.go
index f20baba..7cd0d6f 100644
--- a/cmd/buildstats/buildstats.go
+++ b/cmd/buildstats/buildstats.go
@@ -10,6 +10,7 @@
import (
"context"
"flag"
+ "fmt"
"log"
"golang.org/x/build/buildenv"
@@ -17,7 +18,7 @@
)
var (
- doSync = flag.Bool("sync", false, "sync build stats data from Datastore to BigQuery")
+ mode = flag.String("mode", "", "one of 'sync', 'testspeed'")
verbose = flag.Bool("v", false, "verbose")
)
@@ -27,19 +28,39 @@
buildenv.RegisterFlags()
flag.Parse()
buildstats.Verbose = *verbose
+ if *mode == "" {
+ log.Printf("missing required --mode")
+ flag.Usage()
+ }
env = buildenv.FromFlags()
ctx := context.Background()
- if *doSync {
+ switch *mode {
+ case "sync":
if err := buildstats.SyncBuilds(ctx, env); err != nil {
log.Fatalf("SyncBuilds: %v", err)
}
if err := buildstats.SyncSpans(ctx, env); err != nil {
log.Fatalf("SyncSpans: %v", err)
}
- } else {
- log.Fatalf("the buildstats command doesn't yet do anything except the --sync mode")
+ case "testspeed":
+ ts, err := buildstats.QueryTestStats(ctx, env)
+ if err != nil {
+ log.Fatalf("QueryTestStats: %v", err)
+ }
+ for _, builder := range ts.Builders() {
+ bs := ts.BuilderTestStats[builder]
+ for _, test := range bs.Tests() {
+ fmt.Printf("%s\t%s\t%.1f\t%d\n",
+ builder,
+ test,
+ bs.MedianDuration[test].Seconds(),
+ bs.Runs[test])
+ }
+ }
+ default:
+ log.Fatalf("unknown --mode=%s", *mode)
}
}
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
index 9435c92..d7aea8b 100644
--- a/cmd/coordinator/coordinator.go
+++ b/cmd/coordinator/coordinator.go
@@ -19,7 +19,6 @@
"crypto/rand"
"crypto/sha1"
"crypto/tls"
- "encoding/csv"
"encoding/json"
"errors"
"flag"
@@ -38,7 +37,6 @@
"path"
"runtime"
"sort"
- "strconv"
"strings"
"sync"
"sync/atomic"
@@ -58,6 +56,8 @@
"golang.org/x/build/dashboard"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/buildgo"
+ "golang.org/x/build/internal/buildstats"
+ "golang.org/x/build/internal/singleflight"
"golang.org/x/build/internal/sourcecache"
"golang.org/x/build/livelog"
"golang.org/x/build/maintner/maintnerd/apipb"
@@ -2263,10 +2263,11 @@
// newTestSet returns a new testSet given the dist test names (strings from "go tool dist test -list")
// and benchmark items.
-func (st *buildStatus) newTestSet(distTestNames []string, benchmarks []*buildgo.BenchmarkItem) (*testSet, error) {
+func (st *buildStatus) newTestSet(testStats *buildstats.TestStats, distTestNames []string, benchmarks []*buildgo.BenchmarkItem) (*testSet, error) {
set := &testSet{
st: st,
needsXRepo: map[string]string{},
+ testStats: testStats,
}
for _, name := range distTestNames {
// The misc-vetall builder's "vet/*" tests are special: they require golang.org/x/tools
@@ -2284,7 +2285,7 @@
set.items = append(set.items, &testItem{
set: set,
name: name,
- duration: testDuration(st.BuilderRev.Name, name),
+ duration: testStats.Duration(st.BuilderRev.Name, name),
take: make(chan token, 1),
done: make(chan token),
})
@@ -2295,7 +2296,7 @@
set: set,
name: name,
bench: bench,
- duration: testDuration(st.BuilderRev.Name, name),
+ duration: testStats.Duration(st.BuilderRev.Name, name),
take: make(chan token, 1),
done: make(chan token),
})
@@ -2303,7 +2304,7 @@
return set, nil
}
-func partitionGoTests(builderName string, tests []string) (sets [][]string) {
+func partitionGoTests(testDuration func(string, string) time.Duration, builderName string, tests []string) (sets [][]string) {
var srcTests []string
var cmdTests []string
for _, name := range tests {
@@ -2342,342 +2343,38 @@
return
}
-func secondsToDuration(sec float64) time.Duration {
- return time.Duration(float64(sec) * float64(time.Second))
-}
-
-type testDurationMap map[string]map[string]time.Duration // builder name => test name => avg
-
var (
- testDurations atomic.Value // of testDurationMap
- testDurationsMu sync.Mutex // held while updating testDurations
+ testStats atomic.Value // of *buildstats.TestStats
+ testStatsLoader singleflight.Group
)
-func getTestDurations() testDurationMap {
- if m, ok := testDurations.Load().(testDurationMap); ok {
- return m
+func getTestStats(sl spanlog.Logger) *buildstats.TestStats {
+ sp := sl.CreateSpan("get_test_stats")
+ ts, ok := testStats.Load().(*buildstats.TestStats)
+ if ok && ts.AsOf.After(time.Now().Add(-1*time.Hour)) {
+ sp.Done(nil)
+ return ts
}
- testDurationsMu.Lock()
- defer testDurationsMu.Unlock()
- if m, ok := testDurations.Load().(testDurationMap); ok {
- return m
- }
- updateTestDurationsLocked()
- return testDurations.Load().(testDurationMap)
-}
-
-func updateTestDurations() {
- testDurationsMu.Lock()
- defer testDurationsMu.Unlock()
- updateTestDurationsLocked()
-}
-
-func updateTestDurationsLocked() {
- defer time.AfterFunc(1*time.Hour, updateTestDurations)
- m := loadTestDurations()
- testDurations.Store(m)
-}
-
-// The csv file on cloud storage looks like:
-// Builder,Event,MedianSeconds,count
-// linux-arm-arm5,run_test:runtime:cpu124,334.49922194,10
-// linux-arm,run_test:runtime:cpu124,284.609130993,26
-// linux-arm-arm5,run_test:go_test:cmd/compile/internal/gc,260.0241916,12
-// linux-arm,run_test:go_test:cmd/compile/internal/gc,224.425924681,26
-// solaris-amd64-smartosbuildlet,run_test:test:2_5,199.653975717,9
-// solaris-amd64-smartosbuildlet,run_test:test:1_5,169.89733442,9
-// solaris-amd64-smartosbuildlet,run_test:test:3_5,163.770453839,9
-// solaris-amd64-smartosbuildlet,run_test:test:0_5,158.250119402,9
-// openbsd-386-gce58,run_test:runtime:cpu124,146.494229388,12
-func loadTestDurations() (m testDurationMap) {
- m = make(testDurationMap)
- r, err := storageClient.Bucket(buildEnv.BuildletBucket).Object("test-durations.csv").NewReader(context.Background())
- if err != nil {
- log.Printf("loading test durations object from GCS: %v", err)
- return
- }
- defer r.Close()
- recs, err := csv.NewReader(r).ReadAll()
- if err != nil {
- log.Printf("reading test durations CSV: %v", err)
- return
- }
- for _, rec := range recs {
- if len(rec) < 3 || rec[0] == "Builder" {
- continue
- }
- builder, testName, secondsStr := rec[0], rec[1], rec[2]
- secs, err := strconv.ParseFloat(secondsStr, 64)
+ v, err, _ := testStatsLoader.Do("", func() (interface{}, error) {
+ log.Printf("getTestStats: reloading from BigQuery...")
+ sp := sl.CreateSpan("query_test_stats")
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+ ts, err := buildstats.QueryTestStats(ctx, buildEnv)
+ sp.Done(err)
if err != nil {
- log.Printf("unexpected seconds value in test durations CSV: %v", err)
- continue
+ log.Printf("getTestStats: error: %v", err)
+ return nil, err
}
- mm := m[builder]
- if mm == nil {
- mm = make(map[string]time.Duration)
- m[builder] = mm
- }
- mm[testName] = secondsToDuration(secs)
+ testStats.Store(ts)
+ return ts, nil
+ })
+ if err != nil {
+ sp.Done(err)
+ return nil
}
- return
-}
-
-var minGoTestSpeed = (func() time.Duration {
- var min Seconds
- for name, secs := range fixedTestDuration {
- if !strings.HasPrefix(name, "go_test:") {
- continue
- }
- if min == 0 || secs < min {
- min = secs
- }
- }
- return min.Duration()
-})()
-
-type Seconds float64
-
-func (s Seconds) Duration() time.Duration {
- return time.Duration(float64(s) * float64(time.Second))
-}
-
-// in seconds on Linux/amd64 (once on 2015-05-28), each
-// by themselves. There seems to be a 0.6s+ overhead
-// from the go tool which goes away if they're combined.
-var fixedTestDuration = map[string]Seconds{
- "go_test:archive/tar": 1.30,
- "go_test:archive/zip": 1.68,
- "go_test:bufio": 1.61,
- "go_test:bytes": 1.50,
- "go_test:compress/bzip2": 0.82,
- "go_test:compress/flate": 1.73,
- "go_test:compress/gzip": 0.82,
- "go_test:compress/lzw": 0.86,
- "go_test:compress/zlib": 1.78,
- "go_test:container/heap": 0.69,
- "go_test:container/list": 0.72,
- "go_test:container/ring": 0.64,
- "go_test:crypto/aes": 0.79,
- "go_test:crypto/cipher": 0.96,
- "go_test:crypto/des": 0.96,
- "go_test:crypto/dsa": 0.75,
- "go_test:crypto/ecdsa": 0.86,
- "go_test:crypto/elliptic": 1.06,
- "go_test:crypto/hmac": 0.67,
- "go_test:crypto/md5": 0.77,
- "go_test:crypto/rand": 0.89,
- "go_test:crypto/rc4": 0.71,
- "go_test:crypto/rsa": 1.17,
- "go_test:crypto/sha1": 0.75,
- "go_test:crypto/sha256": 0.68,
- "go_test:crypto/sha512": 0.67,
- "go_test:crypto/subtle": 0.56,
- "go_test:crypto/tls": 3.29,
- "go_test:crypto/x509": 2.81,
- "go_test:database/sql": 1.75,
- "go_test:database/sql/driver": 0.64,
- "go_test:debug/dwarf": 0.77,
- "go_test:debug/elf": 1.41,
- "go_test:debug/gosym": 1.45,
- "go_test:debug/macho": 0.97,
- "go_test:debug/pe": 0.79,
- "go_test:debug/plan9obj": 0.73,
- "go_test:encoding/ascii85": 0.64,
- "go_test:encoding/asn1": 1.16,
- "go_test:encoding/base32": 0.79,
- "go_test:encoding/base64": 0.82,
- "go_test:encoding/binary": 0.96,
- "go_test:encoding/csv": 0.67,
- "go_test:encoding/gob": 2.70,
- "go_test:encoding/hex": 0.66,
- "go_test:encoding/json": 2.20,
- "test:errors": 0.54,
- "go_test:expvar": 1.36,
- "go_test:flag": 0.92,
- "go_test:fmt": 2.02,
- "go_test:go/ast": 1.44,
- "go_test:go/build": 1.42,
- "go_test:go/constant": 0.92,
- "go_test:go/doc": 1.51,
- "go_test:go/format": 0.73,
- "go_test:go/internal/gcimporter": 1.30,
- "go_test:go/parser": 1.30,
- "go_test:go/printer": 1.61,
- "go_test:go/scanner": 0.89,
- "go_test:go/token": 0.92,
- "go_test:go/types": 5.24,
- "go_test:hash/adler32": 0.62,
- "go_test:hash/crc32": 0.68,
- "go_test:hash/crc64": 0.55,
- "go_test:hash/fnv": 0.66,
- "go_test:html": 0.74,
- "go_test:html/template": 1.93,
- "go_test:image": 1.42,
- "go_test:image/color": 0.77,
- "go_test:image/draw": 1.32,
- "go_test:image/gif": 1.15,
- "go_test:image/jpeg": 1.32,
- "go_test:image/png": 1.23,
- "go_test:index/suffixarray": 0.79,
- "go_test:internal/singleflight": 0.66,
- "go_test:io": 0.97,
- "go_test:io/ioutil": 0.73,
- "go_test:log": 0.72,
- "go_test:log/syslog": 2.93,
- "go_test:math": 1.59,
- "go_test:math/big": 3.75,
- "go_test:math/cmplx": 0.81,
- "go_test:math/rand": 1.15,
- "go_test:mime": 1.01,
- "go_test:mime/multipart": 1.51,
- "go_test:mime/quotedprintable": 0.95,
- "go_test:net": 6.71,
- "go_test:net/http": 9.41,
- "go_test:net/http/cgi": 2.00,
- "go_test:net/http/cookiejar": 1.51,
- "go_test:net/http/fcgi": 1.43,
- "go_test:net/http/httptest": 1.36,
- "go_test:net/http/httputil": 1.54,
- "go_test:net/http/internal": 0.68,
- "go_test:net/internal/socktest": 0.58,
- "go_test:net/mail": 0.92,
- "go_test:net/rpc": 1.95,
- "go_test:net/rpc/jsonrpc": 1.50,
- "go_test:net/smtp": 1.43,
- "go_test:net/textproto": 1.01,
- "go_test:net/url": 1.45,
- "go_test:os": 1.88,
- "go_test:os/exec": 2.13,
- "go_test:os/signal": 4.22,
- "go_test:os/user": 0.93,
- "go_test:path": 0.68,
- "go_test:path/filepath": 1.14,
- "go_test:reflect": 3.42,
- "go_test:regexp": 1.65,
- "go_test:regexp/syntax": 1.40,
- "go_test:runtime": 21.02,
- "go_test:runtime/debug": 0.79,
- "go_test:runtime/pprof": 8.01,
- "go_test:sort": 0.96,
- "go_test:strconv": 1.60,
- "go_test:strings": 1.51,
- "go_test:sync": 1.05,
- "go_test:sync/atomic": 1.13,
- "go_test:syscall": 1.69,
- "go_test:testing": 3.70,
- "go_test:testing/quick": 0.74,
- "go_test:text/scanner": 0.79,
- "go_test:text/tabwriter": 0.71,
- "go_test:text/template": 1.65,
- "go_test:text/template/parse": 1.25,
- "go_test:time": 4.20,
- "go_test:unicode": 0.68,
- "go_test:unicode/utf16": 0.77,
- "go_test:unicode/utf8": 0.71,
- "go_test:cmd/addr2line": 1.73,
- "go_test:cmd/api": 1.33,
- "go_test:cmd/asm/internal/asm": 1.24,
- "go_test:cmd/asm/internal/lex": 0.91,
- "go_test:cmd/compile/internal/big": 5.26,
- "go_test:cmd/cover": 3.32,
- "go_test:cmd/fix": 1.26,
- "go_test:cmd/go": 36,
- "go_test:cmd/gofmt": 1.06,
- "go_test:cmd/internal/goobj": 0.65,
- "go_test:cmd/internal/obj": 1.16,
- "go_test:cmd/internal/obj/x86": 1.04,
- "go_test:cmd/internal/rsc.io/arm/armasm": 1.92,
- "go_test:cmd/internal/rsc.io/x86/x86asm": 2.22,
- "go_test:cmd/newlink": 1.48,
- "go_test:cmd/nm": 1.84,
- "go_test:cmd/objdump": 3.60,
- "go_test:cmd/pack": 2.64,
- "go_test:cmd/pprof/internal/profile": 1.29,
- "go_test:cmd/compile/internal/gc": 18,
- "gp_test:cmd/compile/internal/ssa": 8,
- "runtime:cpu124": 44.78,
- "sync_cpu": 1.01,
- "cgo_stdio": 1.53,
- "cgo_life": 1.56,
- "cgo_test": 45.60,
- "race": 42.55,
- "testgodefs": 2.37,
- "testso": 2.72,
- "testcarchive": 11.11,
- "testcshared": 15.80,
- "testshared": 7.13,
- "testasan": 2.56,
- "cgo_errors": 7.03,
- "testsigfwd": 2.74,
- "doc_progs": 5.38,
- "wiki": 3.56,
- "shootout": 11.34,
- "bench_go1": 3.72,
- "test:0_5": 10,
- "test:1_5": 10,
- "test:2_5": 10,
- "test:3_5": 10,
- "test:4_5": 10,
- "codewalk": 2.42,
- "api": 7.38,
-
- "go_test_bench:compress/bzip2": 3.059513602,
- "go_test_bench:image/jpeg": 3.143345345,
- "go_test_bench:encoding/hex": 3.182452293,
- "go_test_bench:expvar": 3.490162906,
- "go_test_bench:crypto/cipher": 3.609317114,
- "go_test_bench:compress/lzw": 3.628982201,
- "go_test_bench:database/sql": 3.693163398,
- "go_test_bench:math/rand": 3.807438591,
- "go_test_bench:bufio": 3.882166683,
- "go_test_bench:context": 4.038173785,
- "go_test_bench:hash/crc32": 4.107135055,
- "go_test_bench:unicode/utf8": 4.205641826,
- "go_test_bench:regexp/syntax": 4.587359311,
- "go_test_bench:sort": 4.660599666,
- "go_test_bench:math/cmplx": 5.311264213,
- "go_test_bench:encoding/gob": 5.326788419,
- "go_test_bench:reflect": 5.777081055,
- "go_test_bench:image/png": 6.12439885,
- "go_test_bench:html/template": 6.765132418,
- "go_test_bench:fmt": 7.476528843,
- "go_test_bench:sync": 7.526458261,
- "go_test_bench:archive/zip": 7.782424696,
- "go_test_bench:regexp": 8.428459563,
- "go_test_bench:image/draw": 8.666510786,
- "go_test_bench:strings": 10.836201759,
- "go_test_bench:time": 10.952476479,
- "go_test_bench:image/gif": 11.373276098,
- "go_test_bench:encoding/json": 11.547950173,
- "go_test_bench:crypto/tls": 11.548834754,
- "go_test_bench:strconv": 12.819669296,
- "go_test_bench:math": 13.7889302,
- "go_test_bench:net": 14.845086695,
- "go_test_bench:net/http": 15.288519219,
- "go_test_bench:bytes": 15.809308703,
- "go_test_bench:index/suffixarray": 23.69239388,
- "go_test_bench:compress/flate": 26.906228664,
- "go_test_bench:math/big": 28.82127674,
-}
-
-// testDuration predicts how long the dist test 'name' will take 'name' will take.
-// It's only a scheduling guess.
-func testDuration(builderName, testName string) time.Duration {
- if false { // disabled for now. never tested. TODO: test, enable.
- durs := getTestDurations()
- bdur := durs[builderName]
- if d, ok := bdur[testName]; ok {
- return d
- }
- }
- if secs, ok := fixedTestDuration[testName]; ok {
- return secs.Duration()
- }
- if strings.HasPrefix(testName, "bench:") {
- // Assume benchmarks are roughly 20 seconds per run.
- return 2 * 5 * 20 * time.Second
- }
- return minGoTestSpeed * 2
+ sp.Done(nil)
+ return v.(*buildstats.TestStats)
}
func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
@@ -2886,7 +2583,10 @@
benches = b
}
}
- set, err := st.newTestSet(testNames, benches)
+
+ testStats := getTestStats(st)
+
+ set, err := st.newTestSet(testStats, testNames, benches)
if err != nil {
return nil, err
}
@@ -3240,8 +2940,9 @@
}
type testSet struct {
- st *buildStatus
- items []*testItem
+ st *buildStatus
+ items []*testItem
+ testStats *buildstats.TestStats
// needsXRepo is the set of x/$REPO repos needed in $GOPATH
// and which git rev that repo should be fetched at.
@@ -3310,7 +3011,7 @@
// First do the go_test:* ones. partitionGoTests
// only returns those, which are the ones we merge together.
- stdSets := partitionGoTests(s.st.BuilderRev.Name, names)
+ stdSets := partitionGoTests(s.testStats.Duration, s.st.BuilderRev.Name, names)
for _, set := range stdSets {
tis := make([]*testItem, len(set))
for i, name := range set {
diff --git a/cmd/coordinator/coordinator_test.go b/cmd/coordinator/coordinator_test.go
index 61ae7a0..0ed8962 100644
--- a/cmd/coordinator/coordinator_test.go
+++ b/cmd/coordinator/coordinator_test.go
@@ -10,6 +10,7 @@
"io/ioutil"
"net/http"
"net/http/httptest"
+ "reflect"
"strings"
"testing"
"time"
@@ -18,14 +19,47 @@
"golang.org/x/build/maintner/maintnerd/apipb"
)
+type Seconds float64
+
+func (s Seconds) Duration() time.Duration {
+ return time.Duration(float64(s) * float64(time.Second))
+}
+
+var fixedTestDuration = map[string]Seconds{
+ "go_test:a": 1,
+ "go_test:b": 1.5,
+ "go_test:c": 2,
+ "go_test:d": 2.50,
+ "go_test:e": 3,
+ "go_test:f": 3.5,
+ "go_test:g": 4,
+ "go_test:h": 4.5,
+ "go_test:i": 5,
+ "go_test:j": 5.5,
+ "go_test:k": 6.5,
+}
+
func TestPartitionGoTests(t *testing.T) {
var in []string
for name := range fixedTestDuration {
in = append(in, name)
}
- sets := partitionGoTests("", in)
- for i, set := range sets {
- t.Logf("set %d = \"-run=^(%s)$\"", i, strings.Join(set, "|"))
+ testDuration := func(builder, testName string) time.Duration {
+ if s, ok := fixedTestDuration[testName]; ok {
+ return s.Duration()
+ }
+ return 3 * time.Second
+ }
+ sets := partitionGoTests(testDuration, "", in)
+ want := [][]string{
+ {"go_test:a", "go_test:b", "go_test:c", "go_test:d", "go_test:e"},
+ {"go_test:f", "go_test:g"},
+ {"go_test:h", "go_test:i"},
+ {"go_test:j"},
+ {"go_test:k"},
+ }
+ if !reflect.DeepEqual(sets, want) {
+ t.Errorf(" got: %v\nwant: %v", sets, want)
}
}
diff --git a/internal/buildstats/buildstats.go b/internal/buildstats/buildstats.go
index 799744a..376eb41 100644
--- a/internal/buildstats/buildstats.go
+++ b/internal/buildstats/buildstats.go
@@ -10,6 +10,8 @@
"fmt"
"log"
"reflect"
+ "sort"
+ "strings"
"time"
"cloud.google.com/go/bigquery"
@@ -309,3 +311,124 @@
}
}
}
+
+// TestStats describes stats for a cmd/dist test on a particular build
+// configuration (a "builder").
+type TestStats struct {
+ // AsOf is the time that the stats were queried from BigQuery.
+ AsOf time.Time
+
+ // BuilderTestStats maps from a builder name to that builder's
+ // test stats.
+ BuilderTestStats map[string]*BuilderTestStats
+}
+
+// Duration returns the median time to run testName on builder, if known.
+// Otherwise it returns some non-zero default value.
+func (ts *TestStats) Duration(builder, testName string) time.Duration {
+ if ts != nil {
+ if bs, ok := ts.BuilderTestStats[builder]; ok {
+ if d, ok := bs.MedianDuration[testName]; ok {
+ return d
+ }
+ }
+ }
+ return 3 * time.Second // some arbitrary value if unknown
+}
+
+func (ts *TestStats) Builders() []string {
+ s := make([]string, 0, len(ts.BuilderTestStats))
+ for k := range ts.BuilderTestStats {
+ s = append(s, k)
+ }
+ sort.Strings(s)
+ return s
+}
+
+type BuilderTestStats struct {
+ // Builder is which build configuration this is for.
+ Builder string
+
+ // Runs is how many times tests have run recently, for some
+ // fuzzy definition of "recently".
+ // The map key is a cmd/dist test name.
+ Runs map[string]int
+
+ // MedianDuration is the median duration for a test to
+ // pass on this BuilderTestStat's Builder.
+ // The map key is a cmd/dist test name.
+ MedianDuration map[string]time.Duration
+}
+
+func (ts *BuilderTestStats) Tests() []string {
+ s := make([]string, 0, len(ts.Runs))
+ for k := range ts.Runs {
+ s = append(s, k)
+ }
+ sort.Strings(s)
+ return s
+}
+
+// QueryTestStats returns stats on all tests for all builders.
+func QueryTestStats(ctx context.Context, env *buildenv.Environment) (*TestStats, error) {
+ ts := &TestStats{
+ AsOf: time.Now(),
+ BuilderTestStats: map[string]*BuilderTestStats{},
+ }
+ bq, err := bigquery.NewClient(ctx, env.ProjectName)
+ if err != nil {
+ return nil, err
+ }
+ defer bq.Close()
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ q := bq.Query(`
+SELECT
+ Builder, Event, APPROX_QUANTILES(Seconds, 100)[OFFSET(50)] as MedianSec, COUNT(*) as N
+FROM
+ builds.Spans
+WHERE
+ Error='' AND
+ StartTime > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 500 HOUR)
+ AND Repo = "go"
+ AND Event LIKE 'run_test:%'
+GROUP BY 1, 2
+`)
+ it, err := q.Read(ctx)
+ if err != nil {
+ return nil, err
+ }
+ n := 0
+ for {
+ var row struct {
+ Builder string
+ Event string
+ MedianSec float64
+ N int
+ }
+ err := it.Next(&row)
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ n++
+ if n > 50000 {
+ break
+ }
+ bs := ts.BuilderTestStats[row.Builder]
+ if bs == nil {
+ bs = &BuilderTestStats{
+ Builder: row.Builder,
+ Runs: map[string]int{},
+ MedianDuration: map[string]time.Duration{},
+ }
+ ts.BuilderTestStats[row.Builder] = bs
+ }
+ distTest := strings.TrimPrefix(row.Event, "run_test:")
+ bs.Runs[distTest] = row.N
+ bs.MedianDuration[distTest] = time.Duration(row.MedianSec * 1e9)
+ }
+ return ts, nil
+}
