cherry/testtiming: add a tool to query test timing data from LUCI
The LUCI query code is essentially copied from watchflakes.
Change-Id: I811d21b921f9cd8f6558a0637a1a162df77edeab
Reviewed-on: https://go-review.googlesource.com/c/scratch/+/598594
TryBot-Bypass: Cherry Mui <cherryyz@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
diff --git a/cherry/testtiming/go.mod b/cherry/testtiming/go.mod
new file mode 100644
index 0000000..bc02362
--- /dev/null
+++ b/cherry/testtiming/go.mod
@@ -0,0 +1,23 @@
+module golang.org/x/scratch/cherry/testtiming
+
+go 1.23
+
+require (
+ go.chromium.org/luci v0.0.0-20240716011143-b5eb7a221b66
+ golang.org/x/sync v0.7.0
+ google.golang.org/protobuf v1.34.2
+)
+
+require (
+ github.com/golang/mock v1.6.0 // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/julienschmidt/httprouter v1.3.0 // indirect
+ github.com/klauspost/compress v1.17.8 // indirect
+ golang.org/x/net v0.23.0 // indirect
+ golang.org/x/sys v0.18.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
+ google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect
+ google.golang.org/grpc v1.61.0 // indirect
+)
diff --git a/cherry/testtiming/go.sum b/cherry/testtiming/go.sum
new file mode 100644
index 0000000..3f62d76
--- /dev/null
+++ b/cherry/testtiming/go.sum
@@ -0,0 +1,61 @@
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/smarty/assertions v1.15.1 h1:812oFiXI+G55vxsFf+8bIZ1ux30qtkdqzKbEFwyX3Tk=
+github.com/smarty/assertions v1.15.1/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
+github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
+github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.chromium.org/luci v0.0.0-20240716011143-b5eb7a221b66 h1:7M08VAaHGjcRE9gonYdmvUL8PKKek9NUZ7mFSjZyJA4=
+go.chromium.org/luci v0.0.0-20240716011143-b5eb7a221b66/go.mod h1:VKWpjBb/iM+b62Tkkvb8Fs6bKxixITrPUpuImWvecvY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y=
+google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
+google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 h1:4++qSzdWBUy9/2x8L5KZgwZw+mjJZ2yDSCGMVM0YzRs=
+google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:PVreiBMirk8ypES6aw9d4p6iiBNSIfZEBqr3UGoAi2E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk=
+google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
+google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
diff --git a/cherry/testtiming/luci.go b/cherry/testtiming/luci.go
new file mode 100644
index 0000000..a399a49
--- /dev/null
+++ b/cherry/testtiming/luci.go
@@ -0,0 +1,477 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// An ad-hoc tool to query test timing data from LUCI.
+//
+// Output CSV with the following columns:
+//
+// commit hash, commit time, [builder,] status, pass duration, fail duration
+//
+// The "builder" column is omitted if only one builder
+// is queried (the -builder flag).
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "log"
+ "net/http"
+ "regexp"
+ "slices"
+ "strings"
+ "time"
+
+ bbpb "go.chromium.org/luci/buildbucket/proto"
+ "go.chromium.org/luci/common/api/gitiles"
+ gpb "go.chromium.org/luci/common/proto/gitiles"
+ "go.chromium.org/luci/grpc/prpc"
+ rdbpb "go.chromium.org/luci/resultdb/proto/v1"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/protobuf/types/known/fieldmaskpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const resultDBHost = "results.api.cr.dev"
+const crBuildBucketHost = "cr-buildbucket.appspot.com"
+const gitilesHost = "go.googlesource.com"
+
+// LUCIClient is a LUCI client.
+type LUCIClient struct {
+ HTTPClient *http.Client
+ GitilesClient gpb.GitilesClient
+ BuildsClient bbpb.BuildsClient
+ BuildersClient bbpb.BuildersClient
+ ResultDBClient rdbpb.ResultDBClient
+
+ // TraceSteps controls whether to log each step name as it's executed.
+ TraceSteps bool
+
+ nProc int
+}
+
+// NewLUCIClient creates a LUCI client.
+// nProc controls concurrency. NewLUCIClient panics if nProc is non-positive.
+func NewLUCIClient(nProc int) *LUCIClient {
+ if nProc < 1 {
+ panic(fmt.Errorf("nProc is %d, want 1 or higher", nProc))
+ }
+ c := new(http.Client)
+ gitilesClient, err := gitiles.NewRESTClient(c, gitilesHost, false)
+ if err != nil {
+ log.Fatal(err)
+ }
+ buildsClient := bbpb.NewBuildsClient(&prpc.Client{
+ C: c,
+ Host: crBuildBucketHost,
+ })
+ buildersClient := bbpb.NewBuildersClient(&prpc.Client{
+ C: c,
+ Host: crBuildBucketHost,
+ })
+ resultDBClient := rdbpb.NewResultDBClient(&prpc.Client{
+ C: c,
+ Host: resultDBHost,
+ })
+ return &LUCIClient{
+ HTTPClient: c,
+ GitilesClient: gitilesClient,
+ BuildsClient: buildsClient,
+ BuildersClient: buildersClient,
+ ResultDBClient: resultDBClient,
+ nProc: nProc,
+ }
+}
+
+type BuilderConfigProperties struct {
+ Repo string `json:"project,omitempty"`
+ GoBranch string `json:"go_branch,omitempty"`
+ Target struct {
+ GOARCH string `json:"goarch,omitempty"`
+ GOOS string `json:"goos,omitempty"`
+ } `json:"target"`
+ KnownIssue int `json:"known_issue,omitempty"`
+}
+
+type Builder struct {
+ Name string
+ *BuilderConfigProperties
+}
+
+type BuildResult struct {
+ ID int64
+ Status bbpb.Status
+ Commit string // commit hash
+ Time time.Time // commit time
+ GoCommit string // for subrepo build, go commit hash
+ BuildTime time.Time // build end time
+ Builder string
+ *BuilderConfigProperties
+ InvocationID string // ResultDB invocation ID
+ LogURL string // textual log of the whole run
+ LogText string
+ StepLogURL string // textual log of the (last) failed step, if any
+ StepLogText string
+ Failures []*Failure
+}
+
+type Commit struct {
+ Hash string
+ Time time.Time
+}
+
+type Project struct {
+ Repo string
+ GoBranch string
+}
+
+type Dashboard struct {
+ Project
+ Builders []Builder
+ Commits []Commit
+ Results [][]*BuildResult // indexed by builder, then by commit
+}
+
+type Failure struct {
+ TestID string
+ Status rdbpb.TestStatus
+ LogURL string
+ LogText string
+}
+
+// ListCommits fetches the list of commits from Gerrit.
+func (c *LUCIClient) ListCommits(ctx context.Context, repo, goBranch string, since time.Time) []Commit {
+ if c.TraceSteps {
+ log.Println("ListCommits", repo, goBranch)
+ }
+ branch := "master"
+ if repo == "go" {
+ branch = goBranch
+ }
+ var commits []Commit
+ var pageToken string
+nextPage:
+ resp, err := c.GitilesClient.Log(ctx, &gpb.LogRequest{
+ Project: repo,
+ Committish: "refs/heads/" + branch,
+ PageSize: 1000,
+ PageToken: pageToken,
+ })
+ if err != nil {
+ log.Fatal(err)
+ }
+ for _, c := range resp.GetLog() {
+ commitTime := c.GetCommitter().GetTime().AsTime()
+ if commitTime.Before(since) {
+ goto done
+ }
+ commits = append(commits, Commit{
+ Hash: c.GetId(),
+ Time: commitTime,
+ })
+ }
+ if resp.GetNextPageToken() != "" {
+ pageToken = resp.GetNextPageToken()
+ goto nextPage
+ }
+done:
+ return commits
+}
+
+// ListBuilders fetches the list of builders, on the given repo and goBranch.
+// If repo and goBranch are empty, it fetches all builders.
+func (c *LUCIClient) ListBuilders(ctx context.Context, repo, goBranch, builder string) ([]Builder, error) {
+ if c.TraceSteps {
+ log.Println("ListBuilders", repo, goBranch)
+ }
+ all := repo == "" && goBranch == ""
+ var builders []Builder
+ var pageToken string
+nextPage:
+ resp, err := c.BuildersClient.ListBuilders(ctx, &bbpb.ListBuildersRequest{
+ Project: "golang",
+ Bucket: "ci",
+ PageSize: 1000,
+ PageToken: pageToken,
+ })
+ if err != nil {
+ return nil, err
+ }
+ for _, b := range resp.GetBuilders() {
+ var p BuilderConfigProperties
+ json.Unmarshal([]byte(b.GetConfig().GetProperties()), &p)
+ if all || (p.Repo == repo && p.GoBranch == goBranch) {
+ bName := b.GetId().GetBuilder()
+ if builder != "" && bName != builder { // just want one builder, skip others
+ continue
+ }
+ builders = append(builders, Builder{bName, &p})
+ }
+ }
+ if resp.GetNextPageToken() != "" {
+ pageToken = resp.GetNextPageToken()
+ goto nextPage
+ }
+ slices.SortFunc(builders, func(a, b Builder) int {
+ return strings.Compare(a.Name, b.Name)
+ })
+ return builders, nil
+}
+
+// GetBuilds fetches builds from one builder.
+func (c *LUCIClient) GetBuilds(ctx context.Context, builder string, since time.Time) ([]*bbpb.Build, error) {
+ if c.TraceSteps {
+ log.Println("GetBuilds", builder)
+ }
+ pred := &bbpb.BuildPredicate{
+ Builder: &bbpb.BuilderID{Project: "golang", Bucket: "ci", Builder: builder},
+ CreateTime: &bbpb.TimeRange{StartTime: timestamppb.New(since)},
+ }
+ mask, err := fieldmaskpb.New((*bbpb.Build)(nil), "id", "builder", "output", "status", "steps", "infra", "end_time")
+ if err != nil {
+ return nil, err
+ }
+ var builds []*bbpb.Build
+ var pageToken string
+nextPage:
+ resp, err := c.BuildsClient.SearchBuilds(ctx, &bbpb.SearchBuildsRequest{
+ Predicate: pred,
+ Mask: &bbpb.BuildMask{Fields: mask},
+ PageSize: 1000,
+ PageToken: pageToken,
+ })
+ if err != nil {
+ return nil, err
+ }
+ builds = append(builds, resp.GetBuilds()...)
+ if resp.GetNextPageToken() != "" {
+ pageToken = resp.GetNextPageToken()
+ goto nextPage
+ }
+ return builds, nil
+}
+
+// ReadBoard reads the build dashboard dash, then fills in the content.
+func (c *LUCIClient) ReadBoard(ctx context.Context, dash *Dashboard, builder string, since time.Time) error {
+ if c.TraceSteps {
+ log.Println("ReadBoard", dash.Repo, dash.GoBranch)
+ }
+ dash.Commits = c.ListCommits(ctx, dash.Repo, dash.GoBranch, since)
+ var err error
+ dash.Builders, err = c.ListBuilders(ctx, dash.Repo, dash.GoBranch, builder)
+ if err != nil {
+ return err
+ }
+
+ dashMap := make([]map[string]*BuildResult, len(dash.Builders)) // indexed by builder, then keyed by commit hash
+
+ // Get builds from builders.
+ g, groupContext := errgroup.WithContext(ctx)
+ g.SetLimit(c.nProc)
+ for i, builder := range dash.Builders {
+ builder := builder
+ buildMap := make(map[string]*BuildResult)
+ dashMap[i] = buildMap
+ g.Go(func() error {
+ bName := builder.Name
+ builds, err := c.GetBuilds(groupContext, bName, since)
+ if err != nil {
+ return err
+ }
+ for _, b := range builds {
+ id := b.GetId()
+ var commit, goCommit string
+ prop := b.GetOutput().GetProperties().GetFields()
+ for _, s := range prop["sources"].GetListValue().GetValues() {
+ x := s.GetStructValue().GetFields()["gitilesCommit"].GetStructValue().GetFields()
+ c := x["id"].GetStringValue()
+ switch repo := x["project"].GetStringValue(); repo {
+ case dash.Repo:
+ commit = c
+ case "go":
+ goCommit = c
+ default:
+ log.Fatalf("repo mismatch: %s %s %s", repo, dash.Repo, buildURL(id))
+ }
+ }
+ if commit == "" {
+ switch b.GetStatus() {
+ case bbpb.Status_SUCCESS:
+ log.Fatalf("empty commit: %s", buildURL(id))
+ default:
+ // unfinished build, or infra failure, ignore
+ continue
+ }
+ }
+ buildTime := b.GetEndTime().AsTime()
+ if r0 := buildMap[commit]; r0 != nil {
+ // A build already exists for the same builder and commit.
+ // Maybe manually retried, or different go commits on same subrepo commit.
+ // Pick the one ended at later time.
+ const printDup = false
+ if printDup {
+ fmt.Printf("skip duplicate build: %s %s %d %d\n", bName, shortHash(commit), id, r0.ID)
+ }
+ if buildTime.Before(r0.BuildTime) {
+ continue
+ }
+ }
+ rdb := b.GetInfra().GetResultdb()
+ if rdb.GetHostname() != resultDBHost {
+ log.Fatalf("ResultDB host mismatch: %s %s %s", rdb.GetHostname(), resultDBHost, buildURL(id))
+ }
+ if b.GetBuilder().GetBuilder() != bName { // sanity check
+ log.Fatalf("builder mismatch: %s %s %s", b.GetBuilder().GetBuilder(), bName, buildURL(id))
+ }
+ r := &BuildResult{
+ ID: id,
+ Status: b.GetStatus(),
+ Commit: commit,
+ GoCommit: goCommit,
+ BuildTime: buildTime,
+ Builder: bName,
+ BuilderConfigProperties: builder.BuilderConfigProperties,
+ InvocationID: rdb.GetInvocation(),
+ }
+ if r.Status == bbpb.Status_FAILURE {
+ links := prop["failure"].GetStructValue().GetFields()["links"].GetListValue().GetValues()
+ for _, l := range links {
+ m := l.GetStructValue().GetFields()
+ if strings.Contains(m["name"].GetStringValue(), "(combined output)") {
+ r.LogURL = m["url"].GetStringValue()
+ break
+ }
+ }
+ if r.LogURL == "" {
+ // No log URL, Probably a build failure.
+ // E.g. https://ci.chromium.org/ui/b/8759448820419452721
+ // Use the build's stderr instead.
+ for _, l := range b.GetOutput().GetLogs() {
+ if l.GetName() == "stderr" {
+ r.LogURL = l.GetViewUrl()
+ break
+ }
+ }
+ }
+
+ // Fetch the stderr of the failed step.
+ steps := b.GetSteps()
+ stepLoop:
+ for i := len(steps) - 1; i >= 0; i-- {
+ s := steps[i]
+ if s.GetStatus() == bbpb.Status_FAILURE {
+ for _, l := range s.GetLogs() {
+ if l.GetName() == "stderr" || l.GetName() == "output" {
+ r.StepLogURL = l.GetViewUrl()
+ break stepLoop
+ }
+ }
+ }
+ }
+ }
+ buildMap[commit] = r
+ }
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return err
+ }
+
+ // Gather into dashboard.
+ dash.Results = make([][]*BuildResult, len(dash.Builders))
+ for i, m := range dashMap {
+ dash.Results[i] = make([]*BuildResult, len(dash.Commits))
+ for j, c := range dash.Commits {
+ r := m[c.Hash]
+ if r == nil {
+ continue
+ }
+ r.Time = c.Time // fill in commit time
+ dash.Results[i][j] = r
+ }
+ }
+
+ return nil
+}
+
+func buildURL(buildID int64) string { // keep in sync with buildUrlRE in github.go
+ return fmt.Sprintf("https://ci.chromium.org/b/%d", buildID)
+}
+
+func shortHash(s string) string {
+ if len(s) > 8 {
+ return s[:8]
+ }
+ return s
+}
+
+var (
+ repo = flag.String("repo", "go", "repo name (defualt: \"go\")")
+ branch = flag.String("branch", "master", "branch (defualt: \"master\")")
+ builder = flag.String("builder", "", "builder to query, if unset, query all builders")
+ test = flag.String("test", "", "test name")
+)
+
+func main() {
+ flag.Parse()
+ if *test == "" {
+ flag.Usage()
+ log.Fatal("test name unset")
+ }
+
+ ctx := context.Background()
+ c := NewLUCIClient(1)
+ c.TraceSteps = true
+
+ // LUCI keeps data up to 60 days, so there is no point to go back farther
+ startTime := time.Now().Add(-60 * 24 * time.Hour)
+ dash := &Dashboard{Project: Project{*repo, *branch}}
+ c.ReadBoard(ctx, dash, *builder, startTime)
+
+ printBuilder := func(string) {}
+ if len(dash.Builders) > 1 {
+ printBuilder = func(s string) { fmt.Print(s, ",") }
+ }
+ for i, b := range dash.Builders {
+ for _, r := range dash.Results[i] {
+ if r == nil {
+ continue
+ }
+ if c.TraceSteps {
+ log.Println("QueryTestResultsRequest", b.Name, shortHash(r.Commit), r.Time)
+ }
+ req := &rdbpb.QueryTestResultsRequest{
+ Invocations: []string{r.InvocationID},
+ Predicate: &rdbpb.TestResultPredicate{
+ TestIdRegexp: regexp.QuoteMeta(*test),
+ },
+ }
+ resp, err := c.ResultDBClient.QueryTestResults(ctx, req)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ for _, rr := range resp.GetTestResults() {
+ status := rr.GetStatus()
+ if status == rdbpb.TestStatus_SKIP {
+ continue
+ }
+ dur := rr.GetDuration().AsDuration()
+ fmt.Print(shortHash(r.Commit), ",", r.Time, ",")
+ printBuilder(b.Name)
+ fmt.Print(status, ",")
+ // Split pass and fail results so it is easy to plot them in
+ // different colors.
+ if status == rdbpb.TestStatus_PASS {
+ fmt.Print(dur.Seconds(), ",")
+ } else {
+ fmt.Print(",", dur.Seconds())
+ }
+ fmt.Println()
+ }
+ }
+ }
+}
