blob: 2ad34480525aa8cee7e924090ef720ee386cb84e [file] [log] [blame]
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !go1.23
package buildstats
import (
"context"
"fmt"
"log"
"reflect"
"strings"
"time"
"cloud.google.com/go/bigquery"
"cloud.google.com/go/datastore"
"golang.org/x/build/buildenv"
"golang.org/x/build/types"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
)
// SyncBuilds syncs the datastore "Build" entities to the BigQuery "Builds" table.
// This stores information on each build as a whole, without details.
func SyncBuilds(ctx context.Context, env *buildenv.Environment) error {
bq, err := bigquery.NewClient(ctx, env.ProjectName)
if err != nil {
return err
}
defer bq.Close()
buildsTable := bq.Dataset("builds").Table("Builds")
meta, err := buildsTable.Metadata(ctx)
if ae, ok := err.(*googleapi.Error); ok && ae.Code == 404 {
if Verbose {
log.Printf("Creating table Builds...")
}
err = buildsTable.Create(ctx, nil)
if err == nil {
meta, err = buildsTable.Metadata(ctx)
}
}
if err != nil {
return fmt.Errorf("getting Builds table metadata: %v", err)
}
if Verbose {
log.Printf("buildstats: Builds metadata: %#v", meta)
}
if len(meta.Schema) == 0 {
if Verbose {
log.Printf("buildstats: builds table has empty schema")
}
schema, err := bigquery.InferSchema(types.BuildRecord{})
if err != nil {
return fmt.Errorf("InferSchema: %v", err)
}
blindWrite := ""
meta, err = buildsTable.Update(ctx, bigquery.TableMetadataToUpdate{Schema: schema}, blindWrite)
if err != nil {
return fmt.Errorf("table.Update schema: %v", err)
}
}
if Verbose {
for i, fs := range meta.Schema {
log.Printf(" schema[%v]: %+v", i, fs)
for j, fs := range fs.Schema {
log.Printf(" .. schema[%v]: %+v", j, fs)
}
}
}
q := bq.Query("SELECT MAX(EndTime) FROM builds.Builds")
it, err := q.Read(ctx)
if err != nil {
return fmt.Errorf("Read: %v", err)
}
var values []bigquery.Value
err = it.Next(&values)
if err == iterator.Done {
return fmt.Errorf("No result.")
}
if err != nil {
return fmt.Errorf("Next: %v", err)
}
var since time.Time
switch t := values[0].(type) {
case nil:
// NULL. No rows.
if Verbose {
log.Printf("buildstats: syncing Builds from the beginning")
}
case time.Time:
since = values[0].(time.Time)
default:
return fmt.Errorf("MAX(EndType) = %T: want nil or time.Time", t)
}
if Verbose {
log.Printf("Max is %v (%v)", since, since.Location())
}
ds, err := datastore.NewClient(ctx, env.ProjectName)
if err != nil {
return fmt.Errorf("datastore.NewClient: %v", err)
}
defer ds.Close()
up := buildsTable.Uploader()
if Verbose {
log.Printf("buildstats: Builds max time: %v", since)
}
dsq := datastore.NewQuery("Build")
if !since.IsZero() {
dsq = dsq.Filter("EndTime >", since).Filter("EndTime <", since.Add(24*90*time.Hour))
} else {
// Ignore rows without endtime.
dsq = dsq.Filter("EndTime >", time.Unix(1, 0))
}
dsq = dsq.Order("EndTime")
dsit := ds.Run(ctx, dsq)
var maxPut time.Time
for {
n := 0
var rows []*bigquery.ValuesSaver
for {
var s types.BuildRecord
key, err := dsit.Next(&s)
if err == iterator.Done {
break
}
n++
if err != nil {
return fmt.Errorf("error querying max EndTime: %v", err)
}
if s.EndTime.IsZero() {
return fmt.Errorf("got zero EndTime")
}
var row []bigquery.Value
var putSchema bigquery.Schema
rv := reflect.ValueOf(s)
for _, fs := range meta.Schema {
if fs.Name[0] == '_' {
continue
}
putSchema = append(putSchema, fs)
row = append(row, rv.FieldByName(fs.Name).Interface())
maxPut = s.EndTime
}
rows = append(rows, &bigquery.ValuesSaver{
Schema: putSchema,
InsertID: key.Encode(),
Row: row,
})
if len(rows) == 1000 {
break
}
}
if n == 0 {
return nil
}
err = up.Put(ctx, rows)
log.Printf("buildstats: Build sync put %d rows, up to %v. error = %v", len(rows), maxPut, err)
if err != nil {
return err
}
}
}
// SyncSpans syncs the datastore "Span" entities to the BigQuery "Spans" table.
// These contain the fine-grained timing details of how a build ran.
func SyncSpans(ctx context.Context, env *buildenv.Environment) error {
bq, err := bigquery.NewClient(ctx, env.ProjectName)
if err != nil {
log.Fatal(err)
}
defer bq.Close()
table := bq.Dataset("builds").Table("Spans")
meta, err := table.Metadata(ctx)
if ae, ok := err.(*googleapi.Error); ok && ae.Code == 404 {
log.Printf("Creating table Spans...")
err = table.Create(ctx, nil)
if err == nil {
meta, err = table.Metadata(ctx)
}
}
if err != nil {
return fmt.Errorf("Metadata: %#v", err)
}
if Verbose {
log.Printf("buildstats: Spans metadata: %#v", meta)
}
schema := meta.Schema
if len(schema) == 0 {
if Verbose {
log.Printf("EMPTY SCHEMA")
}
schema, err = bigquery.InferSchema(types.SpanRecord{})
if err != nil {
return fmt.Errorf("InferSchema: %v", err)
}
blindWrite := ""
meta, err := table.Update(ctx, bigquery.TableMetadataToUpdate{Schema: schema}, blindWrite)
if err != nil {
return fmt.Errorf("table.Update schema: %v", err)
}
schema = meta.Schema
}
if Verbose {
for i, fs := range schema {
log.Printf(" schema[%v]: %+v", i, fs)
for j, fs := range fs.Schema {
log.Printf(" .. schema[%v]: %+v", j, fs)
}
}
}
q := bq.Query("SELECT MAX(EndTime) FROM builds.Spans")
it, err := q.Read(ctx)
if err != nil {
return fmt.Errorf("Read: %v", err)
}
var since time.Time
var values []bigquery.Value
if err := it.Next(&values); err != nil {
if err == iterator.Done {
return fmt.Errorf("Expected at least one row fro MAX(EndTime) query; got none.")
}
return fmt.Errorf("Next: %v", err)
}
switch t := values[0].(type) {
case nil:
// NULL. No rows.
log.Printf("starting from the beginning...")
case time.Time:
since = values[0].(time.Time)
default:
return fmt.Errorf("MAX(EndType) = %T: want nil or time.Time", t)
}
if since.IsZero() {
since = time.Unix(1, 0) // arbitrary
}
ds, err := datastore.NewClient(ctx, env.ProjectName)
if err != nil {
return fmt.Errorf("datastore.NewClient: %v", err)
}
defer ds.Close()
up := table.Uploader()
if Verbose {
log.Printf("buildstats: Span max time: %v", since)
}
dsit := ds.Run(ctx, datastore.NewQuery("Span").Filter("EndTime >", since).Order("EndTime"))
var maxPut time.Time
for {
n := 0
var rows []*bigquery.ValuesSaver
for {
var s types.SpanRecord
key, err := dsit.Next(&s)
if err == iterator.Done {
break
}
n++
if err != nil {
log.Fatal(err)
}
if s.EndTime.IsZero() {
return fmt.Errorf("got zero endtime")
}
var row []bigquery.Value
var putSchema bigquery.Schema
rv := reflect.ValueOf(s)
for _, fs := range meta.Schema {
if fs.Name[0] == '_' {
continue
}
putSchema = append(putSchema, fs)
row = append(row, rv.FieldByName(fs.Name).Interface())
maxPut = s.EndTime
}
rows = append(rows, &bigquery.ValuesSaver{
Schema: putSchema,
InsertID: key.Encode(),
Row: row,
})
if len(rows) == 1000 {
break
}
}
if n == 0 {
return nil
}
err = up.Put(ctx, rows)
log.Printf("buildstats: Spans sync put %d rows, up to %v. error = %v", len(rows), maxPut, err)
if err != nil {
return err
}
}
}
// QueryTestStats returns stats on all tests for all builders.
func QueryTestStats(ctx context.Context, env *buildenv.Environment) (*TestStats, error) {
ts := &TestStats{
AsOf: time.Now(),
BuilderTestStats: map[string]*BuilderTestStats{},
}
bq, err := bigquery.NewClient(ctx, env.ProjectName)
if err != nil {
return nil, err
}
defer bq.Close()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
q := bq.Query(`
SELECT
Builder, Event, APPROX_QUANTILES(Seconds, 100)[OFFSET(50)] as MedianSec, COUNT(*) as N
FROM
builds.Spans
WHERE
Error='' AND
StartTime > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 500 HOUR)
AND Repo = "go"
AND Event LIKE 'run_test:%'
GROUP BY 1, 2
`)
it, err := q.Read(ctx)
if err != nil {
return nil, err
}
n := 0
for {
var row struct {
Builder string
Event string
MedianSec float64
N int
}
err := it.Next(&row)
if err == iterator.Done {
break
}
if err != nil {
return nil, err
}
n++
if n > 50000 {
break
}
bs := ts.BuilderTestStats[row.Builder]
if bs == nil {
bs = &BuilderTestStats{
Builder: row.Builder,
Runs: map[string]int{},
MedianDuration: map[string]time.Duration{},
}
ts.BuilderTestStats[row.Builder] = bs
}
distTest := strings.TrimPrefix(row.Event, "run_test:")
bs.Runs[distTest] = row.N
bs.MedianDuration[distTest] = time.Duration(row.MedianSec * 1e9)
}
return ts, nil
}