blob: a399a49c0195ca63770d4e045dc732d34afd2352 [file] [log] [blame]
Cherry Mui6cf087e2024-07-16 12:19:19 -04001// Copyright 2024 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// An ad-hoc tool to query test timing data from LUCI.
6//
7// Output CSV with the following columns:
8//
9// commit hash, commit time, [builder,] status, pass duration, fail duration
10//
11// The "builder" column is omitted if only one builder
12// is queried (the -builder flag).
13package main
14
15import (
16 "context"
17 "encoding/json"
18 "flag"
19 "fmt"
20 "log"
21 "net/http"
22 "regexp"
23 "slices"
24 "strings"
25 "time"
26
27 bbpb "go.chromium.org/luci/buildbucket/proto"
28 "go.chromium.org/luci/common/api/gitiles"
29 gpb "go.chromium.org/luci/common/proto/gitiles"
30 "go.chromium.org/luci/grpc/prpc"
31 rdbpb "go.chromium.org/luci/resultdb/proto/v1"
32 "golang.org/x/sync/errgroup"
33 "google.golang.org/protobuf/types/known/fieldmaskpb"
34 "google.golang.org/protobuf/types/known/timestamppb"
35)
36
37const resultDBHost = "results.api.cr.dev"
38const crBuildBucketHost = "cr-buildbucket.appspot.com"
39const gitilesHost = "go.googlesource.com"
40
41// LUCIClient is a LUCI client.
42type LUCIClient struct {
43 HTTPClient *http.Client
44 GitilesClient gpb.GitilesClient
45 BuildsClient bbpb.BuildsClient
46 BuildersClient bbpb.BuildersClient
47 ResultDBClient rdbpb.ResultDBClient
48
49 // TraceSteps controls whether to log each step name as it's executed.
50 TraceSteps bool
51
52 nProc int
53}
54
55// NewLUCIClient creates a LUCI client.
56// nProc controls concurrency. NewLUCIClient panics if nProc is non-positive.
57func NewLUCIClient(nProc int) *LUCIClient {
58 if nProc < 1 {
59 panic(fmt.Errorf("nProc is %d, want 1 or higher", nProc))
60 }
61 c := new(http.Client)
62 gitilesClient, err := gitiles.NewRESTClient(c, gitilesHost, false)
63 if err != nil {
64 log.Fatal(err)
65 }
66 buildsClient := bbpb.NewBuildsClient(&prpc.Client{
67 C: c,
68 Host: crBuildBucketHost,
69 })
70 buildersClient := bbpb.NewBuildersClient(&prpc.Client{
71 C: c,
72 Host: crBuildBucketHost,
73 })
74 resultDBClient := rdbpb.NewResultDBClient(&prpc.Client{
75 C: c,
76 Host: resultDBHost,
77 })
78 return &LUCIClient{
79 HTTPClient: c,
80 GitilesClient: gitilesClient,
81 BuildsClient: buildsClient,
82 BuildersClient: buildersClient,
83 ResultDBClient: resultDBClient,
84 nProc: nProc,
85 }
86}
87
88type BuilderConfigProperties struct {
89 Repo string `json:"project,omitempty"`
90 GoBranch string `json:"go_branch,omitempty"`
91 Target struct {
92 GOARCH string `json:"goarch,omitempty"`
93 GOOS string `json:"goos,omitempty"`
94 } `json:"target"`
95 KnownIssue int `json:"known_issue,omitempty"`
96}
97
98type Builder struct {
99 Name string
100 *BuilderConfigProperties
101}
102
103type BuildResult struct {
104 ID int64
105 Status bbpb.Status
106 Commit string // commit hash
107 Time time.Time // commit time
108 GoCommit string // for subrepo build, go commit hash
109 BuildTime time.Time // build end time
110 Builder string
111 *BuilderConfigProperties
112 InvocationID string // ResultDB invocation ID
113 LogURL string // textual log of the whole run
114 LogText string
115 StepLogURL string // textual log of the (last) failed step, if any
116 StepLogText string
117 Failures []*Failure
118}
119
120type Commit struct {
121 Hash string
122 Time time.Time
123}
124
125type Project struct {
126 Repo string
127 GoBranch string
128}
129
130type Dashboard struct {
131 Project
132 Builders []Builder
133 Commits []Commit
134 Results [][]*BuildResult // indexed by builder, then by commit
135}
136
137type Failure struct {
138 TestID string
139 Status rdbpb.TestStatus
140 LogURL string
141 LogText string
142}
143
144// ListCommits fetches the list of commits from Gerrit.
145func (c *LUCIClient) ListCommits(ctx context.Context, repo, goBranch string, since time.Time) []Commit {
146 if c.TraceSteps {
147 log.Println("ListCommits", repo, goBranch)
148 }
149 branch := "master"
150 if repo == "go" {
151 branch = goBranch
152 }
153 var commits []Commit
154 var pageToken string
155nextPage:
156 resp, err := c.GitilesClient.Log(ctx, &gpb.LogRequest{
157 Project: repo,
158 Committish: "refs/heads/" + branch,
159 PageSize: 1000,
160 PageToken: pageToken,
161 })
162 if err != nil {
163 log.Fatal(err)
164 }
165 for _, c := range resp.GetLog() {
166 commitTime := c.GetCommitter().GetTime().AsTime()
167 if commitTime.Before(since) {
168 goto done
169 }
170 commits = append(commits, Commit{
171 Hash: c.GetId(),
172 Time: commitTime,
173 })
174 }
175 if resp.GetNextPageToken() != "" {
176 pageToken = resp.GetNextPageToken()
177 goto nextPage
178 }
179done:
180 return commits
181}
182
183// ListBuilders fetches the list of builders, on the given repo and goBranch.
184// If repo and goBranch are empty, it fetches all builders.
185func (c *LUCIClient) ListBuilders(ctx context.Context, repo, goBranch, builder string) ([]Builder, error) {
186 if c.TraceSteps {
187 log.Println("ListBuilders", repo, goBranch)
188 }
189 all := repo == "" && goBranch == ""
190 var builders []Builder
191 var pageToken string
192nextPage:
193 resp, err := c.BuildersClient.ListBuilders(ctx, &bbpb.ListBuildersRequest{
194 Project: "golang",
195 Bucket: "ci",
196 PageSize: 1000,
197 PageToken: pageToken,
198 })
199 if err != nil {
200 return nil, err
201 }
202 for _, b := range resp.GetBuilders() {
203 var p BuilderConfigProperties
204 json.Unmarshal([]byte(b.GetConfig().GetProperties()), &p)
205 if all || (p.Repo == repo && p.GoBranch == goBranch) {
206 bName := b.GetId().GetBuilder()
207 if builder != "" && bName != builder { // just want one builder, skip others
208 continue
209 }
210 builders = append(builders, Builder{bName, &p})
211 }
212 }
213 if resp.GetNextPageToken() != "" {
214 pageToken = resp.GetNextPageToken()
215 goto nextPage
216 }
217 slices.SortFunc(builders, func(a, b Builder) int {
218 return strings.Compare(a.Name, b.Name)
219 })
220 return builders, nil
221}
222
223// GetBuilds fetches builds from one builder.
224func (c *LUCIClient) GetBuilds(ctx context.Context, builder string, since time.Time) ([]*bbpb.Build, error) {
225 if c.TraceSteps {
226 log.Println("GetBuilds", builder)
227 }
228 pred := &bbpb.BuildPredicate{
229 Builder: &bbpb.BuilderID{Project: "golang", Bucket: "ci", Builder: builder},
230 CreateTime: &bbpb.TimeRange{StartTime: timestamppb.New(since)},
231 }
232 mask, err := fieldmaskpb.New((*bbpb.Build)(nil), "id", "builder", "output", "status", "steps", "infra", "end_time")
233 if err != nil {
234 return nil, err
235 }
236 var builds []*bbpb.Build
237 var pageToken string
238nextPage:
239 resp, err := c.BuildsClient.SearchBuilds(ctx, &bbpb.SearchBuildsRequest{
240 Predicate: pred,
241 Mask: &bbpb.BuildMask{Fields: mask},
242 PageSize: 1000,
243 PageToken: pageToken,
244 })
245 if err != nil {
246 return nil, err
247 }
248 builds = append(builds, resp.GetBuilds()...)
249 if resp.GetNextPageToken() != "" {
250 pageToken = resp.GetNextPageToken()
251 goto nextPage
252 }
253 return builds, nil
254}
255
256// ReadBoard reads the build dashboard dash, then fills in the content.
257func (c *LUCIClient) ReadBoard(ctx context.Context, dash *Dashboard, builder string, since time.Time) error {
258 if c.TraceSteps {
259 log.Println("ReadBoard", dash.Repo, dash.GoBranch)
260 }
261 dash.Commits = c.ListCommits(ctx, dash.Repo, dash.GoBranch, since)
262 var err error
263 dash.Builders, err = c.ListBuilders(ctx, dash.Repo, dash.GoBranch, builder)
264 if err != nil {
265 return err
266 }
267
268 dashMap := make([]map[string]*BuildResult, len(dash.Builders)) // indexed by builder, then keyed by commit hash
269
270 // Get builds from builders.
271 g, groupContext := errgroup.WithContext(ctx)
272 g.SetLimit(c.nProc)
273 for i, builder := range dash.Builders {
274 builder := builder
275 buildMap := make(map[string]*BuildResult)
276 dashMap[i] = buildMap
277 g.Go(func() error {
278 bName := builder.Name
279 builds, err := c.GetBuilds(groupContext, bName, since)
280 if err != nil {
281 return err
282 }
283 for _, b := range builds {
284 id := b.GetId()
285 var commit, goCommit string
286 prop := b.GetOutput().GetProperties().GetFields()
287 for _, s := range prop["sources"].GetListValue().GetValues() {
288 x := s.GetStructValue().GetFields()["gitilesCommit"].GetStructValue().GetFields()
289 c := x["id"].GetStringValue()
290 switch repo := x["project"].GetStringValue(); repo {
291 case dash.Repo:
292 commit = c
293 case "go":
294 goCommit = c
295 default:
296 log.Fatalf("repo mismatch: %s %s %s", repo, dash.Repo, buildURL(id))
297 }
298 }
299 if commit == "" {
300 switch b.GetStatus() {
301 case bbpb.Status_SUCCESS:
302 log.Fatalf("empty commit: %s", buildURL(id))
303 default:
304 // unfinished build, or infra failure, ignore
305 continue
306 }
307 }
308 buildTime := b.GetEndTime().AsTime()
309 if r0 := buildMap[commit]; r0 != nil {
310 // A build already exists for the same builder and commit.
311 // Maybe manually retried, or different go commits on same subrepo commit.
312 // Pick the one ended at later time.
313 const printDup = false
314 if printDup {
315 fmt.Printf("skip duplicate build: %s %s %d %d\n", bName, shortHash(commit), id, r0.ID)
316 }
317 if buildTime.Before(r0.BuildTime) {
318 continue
319 }
320 }
321 rdb := b.GetInfra().GetResultdb()
322 if rdb.GetHostname() != resultDBHost {
323 log.Fatalf("ResultDB host mismatch: %s %s %s", rdb.GetHostname(), resultDBHost, buildURL(id))
324 }
325 if b.GetBuilder().GetBuilder() != bName { // sanity check
326 log.Fatalf("builder mismatch: %s %s %s", b.GetBuilder().GetBuilder(), bName, buildURL(id))
327 }
328 r := &BuildResult{
329 ID: id,
330 Status: b.GetStatus(),
331 Commit: commit,
332 GoCommit: goCommit,
333 BuildTime: buildTime,
334 Builder: bName,
335 BuilderConfigProperties: builder.BuilderConfigProperties,
336 InvocationID: rdb.GetInvocation(),
337 }
338 if r.Status == bbpb.Status_FAILURE {
339 links := prop["failure"].GetStructValue().GetFields()["links"].GetListValue().GetValues()
340 for _, l := range links {
341 m := l.GetStructValue().GetFields()
342 if strings.Contains(m["name"].GetStringValue(), "(combined output)") {
343 r.LogURL = m["url"].GetStringValue()
344 break
345 }
346 }
347 if r.LogURL == "" {
348 // No log URL, Probably a build failure.
349 // E.g. https://ci.chromium.org/ui/b/8759448820419452721
350 // Use the build's stderr instead.
351 for _, l := range b.GetOutput().GetLogs() {
352 if l.GetName() == "stderr" {
353 r.LogURL = l.GetViewUrl()
354 break
355 }
356 }
357 }
358
359 // Fetch the stderr of the failed step.
360 steps := b.GetSteps()
361 stepLoop:
362 for i := len(steps) - 1; i >= 0; i-- {
363 s := steps[i]
364 if s.GetStatus() == bbpb.Status_FAILURE {
365 for _, l := range s.GetLogs() {
366 if l.GetName() == "stderr" || l.GetName() == "output" {
367 r.StepLogURL = l.GetViewUrl()
368 break stepLoop
369 }
370 }
371 }
372 }
373 }
374 buildMap[commit] = r
375 }
376 return nil
377 })
378 }
379 if err := g.Wait(); err != nil {
380 return err
381 }
382
383 // Gather into dashboard.
384 dash.Results = make([][]*BuildResult, len(dash.Builders))
385 for i, m := range dashMap {
386 dash.Results[i] = make([]*BuildResult, len(dash.Commits))
387 for j, c := range dash.Commits {
388 r := m[c.Hash]
389 if r == nil {
390 continue
391 }
392 r.Time = c.Time // fill in commit time
393 dash.Results[i][j] = r
394 }
395 }
396
397 return nil
398}
399
400func buildURL(buildID int64) string { // keep in sync with buildUrlRE in github.go
401 return fmt.Sprintf("https://ci.chromium.org/b/%d", buildID)
402}
403
404func shortHash(s string) string {
405 if len(s) > 8 {
406 return s[:8]
407 }
408 return s
409}
410
411var (
412 repo = flag.String("repo", "go", "repo name (defualt: \"go\")")
413 branch = flag.String("branch", "master", "branch (defualt: \"master\")")
414 builder = flag.String("builder", "", "builder to query, if unset, query all builders")
415 test = flag.String("test", "", "test name")
416)
417
418func main() {
419 flag.Parse()
420 if *test == "" {
421 flag.Usage()
422 log.Fatal("test name unset")
423 }
424
425 ctx := context.Background()
426 c := NewLUCIClient(1)
427 c.TraceSteps = true
428
429 // LUCI keeps data up to 60 days, so there is no point to go back farther
430 startTime := time.Now().Add(-60 * 24 * time.Hour)
431 dash := &Dashboard{Project: Project{*repo, *branch}}
432 c.ReadBoard(ctx, dash, *builder, startTime)
433
434 printBuilder := func(string) {}
435 if len(dash.Builders) > 1 {
436 printBuilder = func(s string) { fmt.Print(s, ",") }
437 }
438 for i, b := range dash.Builders {
439 for _, r := range dash.Results[i] {
440 if r == nil {
441 continue
442 }
443 if c.TraceSteps {
444 log.Println("QueryTestResultsRequest", b.Name, shortHash(r.Commit), r.Time)
445 }
446 req := &rdbpb.QueryTestResultsRequest{
447 Invocations: []string{r.InvocationID},
448 Predicate: &rdbpb.TestResultPredicate{
449 TestIdRegexp: regexp.QuoteMeta(*test),
450 },
451 }
452 resp, err := c.ResultDBClient.QueryTestResults(ctx, req)
453 if err != nil {
454 log.Fatal(err)
455 }
456
457 for _, rr := range resp.GetTestResults() {
458 status := rr.GetStatus()
459 if status == rdbpb.TestStatus_SKIP {
460 continue
461 }
462 dur := rr.GetDuration().AsDuration()
463 fmt.Print(shortHash(r.Commit), ",", r.Time, ",")
464 printBuilder(b.Name)
465 fmt.Print(status, ",")
466 // Split pass and fail results so it is easy to plot them in
467 // different colors.
468 if status == rdbpb.TestStatus_PASS {
469 fmt.Print(dur.Seconds(), ",")
470 } else {
471 fmt.Print(",", dur.Seconds())
472 }
473 fmt.Println()
474 }
475 }
476 }
477}