app, gitmirror, maintner: use maintner for dashboard, not datastore
Historically, the build.golang.org was the entire build system, and it
maintained a parallel copy of that git history in its datastore. It
was always buggy and incomplete and things like force pushes were
scary because the datastore mirror could get out of sync. It was also
a lot of code to support that sync.
This changes build.golang.org to instead get the git history from
maintnerd, and then we can remove all the HTTP handlers around
updating it, and can remove all the gitmirror code to call it to
maintain it.
Now build.golang.org only keeps build results, keyed on the commit
hash. It's much less code, but is still an App Engine app for now.
(but it's getting small enough, that porting it to
cloud.google.com/go/datastore is looking very simple)
This also adds a new "repos" package to unify the configuration of the
various Go repos. There were incomplete & redundant copies all over
the place.
Updates golang/go#34744
Fixes golang/go#35828
Fixes golang/go#31236 (New branch=mixed support adds this when desired)
Fixes golang/go#35944
Change-Id: Ifb39417287df3dea052ba8510566d80b4bc75d51
Reviewed-on: https://go-review.googlesource.com/c/build/+/208697
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/app/appengine/Makefile b/app/appengine/Makefile
index 2f8567b..c9d55ba 100644
--- a/app/appengine/Makefile
+++ b/app/appengine/Makefile
@@ -2,6 +2,10 @@
echo "See Makefile for usage"
exit 1
-deploy:
+deploy-prod:
go install golang.org/x/build/cmd/xb
GO111MODULE=on gcloud app --account=$$(xb google-email) --project=golang-org deploy app.yaml
+
+deploy-test:
+ go install golang.org/x/build/cmd/xb
+ GO111MODULE=on gcloud app --account=$$(xb google-email) --project=golang-org deploy --no-promote app.yaml
diff --git a/app/appengine/app.yaml b/app/appengine/app.yaml
index efbdf1e..b1ff185 100644
--- a/app/appengine/app.yaml
+++ b/app/appengine/app.yaml
@@ -2,9 +2,6 @@
service: build
handlers:
- - url: /static
- static_dir: app/appengine/static
- secure: always
- url: /.*
script: auto
secure: always
diff --git a/app/appengine/build.go b/app/appengine/build.go
index 25da662..098c77c 100644
--- a/app/appengine/build.go
+++ b/app/appengine/build.go
@@ -12,8 +12,8 @@
"fmt"
"io"
"io/ioutil"
+ pathpkg "path"
"strings"
- "time"
"golang.org/x/build/dashboard"
"golang.org/x/build/internal/loghash"
@@ -27,10 +27,8 @@
// A Package describes a package that is listed on the dashboard.
type Package struct {
- Kind string // "subrepo", "external", or empty for the main Go tree
- Name string // "Go", "arch", "net", ...
- Path string // empty for the main Go tree, else "golang.org/x/foo"
- NextNum int // Num of the next head Commit
+ Name string // "Go", "arch", "net", ...
+ Path string // empty for the main Go tree, else "golang.org/x/foo"
}
func (p *Package) String() string {
@@ -49,54 +47,73 @@
// not being able to load an entity with old legacy struct fields into
// the Commit type that has since removed those fields.
func filterDatastoreError(err error) error {
- if em, ok := err.(*datastore.ErrFieldMismatch); ok {
- switch em.FieldName {
- case "NeedsBenchmarking", "TryPatch", "FailNotificationSent":
- // Removed in CLs 208397 and 208324.
- return nil
- }
- }
- if me, ok := err.(appengine.MultiError); ok {
- any := false
- for i, err := range me {
- me[i] = filterDatastoreError(err)
- if me[i] != nil {
- any = true
+ return filterAppEngineError(err, func(err error) bool {
+ if em, ok := err.(*datastore.ErrFieldMismatch); ok {
+ switch em.FieldName {
+ case "NeedsBenchmarking", "TryPatch", "FailNotificationSent":
+ // Removed in CLs 208397 and 208324.
+ return true
+ case "PackagePath", "ParentHash", "Num", "User", "Desc", "Time", "Branch", "NextNum":
+ // Removed in move to maintner in CL 208697.
+ return true
}
}
- if !any {
+ return false
+ })
+}
+
+// filterNoSuchEntity returns err, unless it's just about datastore
+// not being able to load an entity because it doesn't exist.
+func filterNoSuchEntity(err error) error {
+ return filterAppEngineError(err, func(err error) bool {
+ return err == datastore.ErrNoSuchEntity
+ })
+}
+
+// filterAppEngineError returns err, unless ignore(err) is true,
+// in which case it returns nil. If err is an appengine.MultiError,
+// it returns either nil (if all errors are ignored) or a deep copy
+// with the non-ignored errors.
+func filterAppEngineError(err error, ignore func(error) bool) error {
+ if err == nil || ignore(err) {
+ return nil
+ }
+ if me, ok := err.(appengine.MultiError); ok {
+ me2 := make(appengine.MultiError, 0, len(me))
+ for _, err := range me {
+ if e2 := filterAppEngineError(err, ignore); e2 != nil {
+ me2 = append(me2, e2)
+ }
+ }
+ if len(me2) == 0 {
return nil
}
+ return me2
}
return err
}
-// LastCommit returns the most recent Commit for this Package.
-func (p *Package) LastCommit(c context.Context) (*Commit, error) {
- var commits []*Commit
- _, err := datastore.NewQuery("Commit").
- Ancestor(p.Key(c)).
- Order("-Time").
- Limit(1).
- GetAll(c, &commits)
+// getOrMakePackageInTx fetches a Package by path from the datastore,
+// creating it if necessary. It should be run in a transaction.
+func getOrMakePackageInTx(c context.Context, path string) (*Package, error) {
+ p := &Package{Path: path}
+ if path != "" {
+ p.Name = pathpkg.Base(path)
+ } else {
+ p.Name = "Go"
+ }
+ err := datastore.Get(c, p.Key(c), p)
err = filterDatastoreError(err)
+ if err == datastore.ErrNoSuchEntity {
+ if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+ return nil, err
+ }
+ return p, nil
+ }
if err != nil {
return nil, err
}
- if len(commits) != 1 {
- return nil, datastore.ErrNoSuchEntity
- }
- return commits[0], nil
-}
-
-// GetPackage fetches a Package by path from the datastore.
-func GetPackage(c context.Context, path string) (*Package, error) {
- p := &Package{Path: path}
- err := datastore.Get(c, p.Key(c), p)
- if err == datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("package %q not found", path)
- }
- return p, err
+ return p, nil
}
type builderAndGoHash struct {
@@ -111,21 +128,12 @@
type Commit struct {
PackagePath string // (empty for main repo commits)
Hash string
- ParentHash string
- Num int // Internal monotonic counter unique to this package.
-
- User string
- Desc string `datastore:",noindex"`
- Time time.Time
- Branch string
// ResultData is the Data string of each build Result for this Commit.
// For non-Go commits, only the Results for the current Go tip, weekly,
// and release Tags are stored here. This is purely de-normalized data.
// The complete data set is stored in Result entities.
ResultData []string `datastore:",noindex"`
-
- buildingURLs map[builderAndGoHash]string
}
func (com *Commit) Key(c context.Context) *datastore.Key {
@@ -137,22 +145,15 @@
return datastore.NewKey(c, "Commit", key, 0, p.Key(c))
}
-func (c *Commit) Valid() error {
- if !validHash(c.Hash) {
- return errors.New("invalid Hash")
- }
- if c.ParentHash != "" && !validHash(c.ParentHash) { // empty is OK
- return errors.New("invalid ParentHash")
- }
- return nil
+// Valid reports whether the commit is valid.
+func (c *Commit) Valid() bool {
+ // Valid really just means the hash is populated.
+ return validHash(c.Hash)
}
func putCommit(c context.Context, com *Commit) error {
- if err := com.Valid(); err != nil {
- return fmt.Errorf("putting Commit: %v", err)
- }
- if com.Num == 0 && com.ParentHash != "0000" { // 0000 is used in tests
- return fmt.Errorf("putting Commit: invalid Num (must be > 0)")
+ if !com.Valid() {
+ return errors.New("putting Commit: commit is not valid")
}
if _, err := datastore.Put(c, com.Key(c), com); err != nil {
return fmt.Errorf("putting Commit: %v", err)
@@ -168,9 +169,13 @@
// It must be called from inside a datastore transaction.
func (com *Commit) AddResult(c context.Context, r *Result) error {
err := datastore.Get(c, com.Key(c), com)
- err = filterDatastoreError(err)
- if err != nil {
- return fmt.Errorf("getting Commit: %v", err)
+ if err == datastore.ErrNoSuchEntity {
+ // If it doesn't exist, we create it below.
+ } else {
+ err = filterDatastoreError(err)
+ if err != nil {
+ return fmt.Errorf("Commit.AddResult, getting Commit: %v", err)
+ }
}
var resultExists bool
@@ -216,17 +221,18 @@
}
// Result returns the build Result for this Commit for the given builder/goHash.
+//
+// For the main Go repo, goHash is the empty string.
func (c *Commit) Result(builder, goHash string) *Result {
- for _, r := range c.ResultData {
- if !strings.HasPrefix(r, builder) {
- // Avoid strings.SplitN alloc in the common case.
- continue
- }
- p := strings.SplitN(r, "|", 4)
- if len(p) != 4 || p[0] != builder || p[3] != goHash {
- continue
- }
- return partsToResult(c, p)
+ return result(c.ResultData, c.Hash, c.PackagePath, builder, goHash)
+}
+
+// Result returns the build Result for this commit for the given builder/goHash.
+//
+// For the main Go repo, goHash is the empty string.
+func (c *CommitInfo) Result(builder, goHash string) *Result {
+ if r := result(c.ResultData, c.Hash, c.PackagePath, builder, goHash); r != nil {
+ return r
}
if u, ok := c.buildingURLs[builderAndGoHash{builder, goHash}]; ok {
return &Result{
@@ -239,6 +245,21 @@
return nil
}
+func result(resultData []string, hash, packagePath, builder, goHash string) *Result {
+ for _, r := range resultData {
+ if !strings.HasPrefix(r, builder) {
+ // Avoid strings.SplitN alloc in the common case.
+ continue
+ }
+ p := strings.SplitN(r, "|", 4)
+ if len(p) != 4 || p[0] != builder || p[3] != goHash {
+ continue
+ }
+ return partsToResult(hash, packagePath, p)
+ }
+ return nil
+}
+
// isUntested reports whether a cell in the build.golang.org grid is
// an untested configuration.
//
@@ -265,18 +286,23 @@
}
// Results returns the build Results for this Commit.
-func (c *Commit) Results() (results []*Result) {
+func (c *CommitInfo) Results() (results []*Result) {
for _, r := range c.ResultData {
p := strings.SplitN(r, "|", 4)
if len(p) != 4 {
continue
}
- results = append(results, partsToResult(c, p))
+ results = append(results, partsToResult(c.Hash, c.PackagePath, p))
}
return
}
-func (c *Commit) ResultGoHashes() []string {
+// ResultGoHashes, for non-go repos, returns the list of Go hashes that
+// this repo has been (or should be) built at.
+//
+// For the main Go repo it always returns a slice with 1 element: the
+// empty string.
+func (c *CommitInfo) ResultGoHashes() []string {
// For the main repo, just return the empty string
// (there's no corresponding main repo hash for a main repo Commit).
// This function is only really useful for sub-repos.
@@ -315,12 +341,12 @@
}
}
-// partsToResult converts a Commit and ResultData substrings to a Result.
-func partsToResult(c *Commit, p []string) *Result {
+// partsToResult creates a Result from ResultData substrings.
+func partsToResult(hash, packagePath string, p []string) *Result {
return &Result{
Builder: p[0],
- Hash: c.Hash,
- PackagePath: c.PackagePath,
+ Hash: hash,
+ PackagePath: packagePath,
GoHash: p[3],
OK: p[1] == "true",
LogHash: p[2],
@@ -331,11 +357,11 @@
//
// Each Result entity is a descendant of its associated Package entity.
type Result struct {
- PackagePath string // (empty for Go commits)
Builder string // "os-arch[-note]"
+ PackagePath string // (empty for Go commits, else "golang.org/x/foo")
Hash string
- // The Go Commit this was built against (empty for Go commits).
+ // The Go Commit this was built against (when PackagePath != ""; empty for Go commits).
GoHash string
BuildingURL string `datastore:"-"` // non-empty if currently building
@@ -396,85 +422,3 @@
_, err = datastore.Put(c, key, &Log{b.Bytes()})
return
}
-
-// A Tag is used to keep track of the most recent Go weekly and release tags.
-// Typically there will be one Tag entity for each kind of git tag.
-type Tag struct {
- Kind string // "release", or "tip"
- Name string // the tag itself (for example: "release.r60")
- Hash string
-}
-
-func (t *Tag) String() string {
- if t.Kind == "tip" {
- return "tip"
- }
- return t.Name
-}
-
-func (t *Tag) Key(c context.Context) *datastore.Key {
- p := &Package{}
- s := t.Kind
- if t.Kind == "release" {
- s += "-" + t.Name
- }
- return datastore.NewKey(c, "Tag", s, 0, p.Key(c))
-}
-
-func (t *Tag) Valid() error {
- if t.Kind != "release" && t.Kind != "tip" {
- return errors.New("invalid Kind")
- }
- if t.Kind == "release" && t.Name == "" {
- return errors.New("release must have Name")
- }
- if !validHash(t.Hash) {
- return errors.New("invalid Hash")
- }
- return nil
-}
-
-// Commit returns the Commit that corresponds with this Tag.
-func (t *Tag) Commit(c context.Context) (*Commit, error) {
- com := &Commit{Hash: t.Hash}
- err := datastore.Get(c, com.Key(c), com)
- err = filterDatastoreError(err)
- return com, err
-}
-
-// GetTag fetches a Tag by name from the datastore.
-func GetTag(c context.Context, kind, name string) (*Tag, error) {
- t := &Tag{Kind: kind, Name: name}
- if err := datastore.Get(c, t.Key(c), t); err != nil {
- return nil, err
- }
- if err := t.Valid(); err != nil {
- return nil, err
- }
- return t, nil
-}
-
-// Packages returns packages of the specified kind.
-// Kind must be one of "external" or "subrepo".
-func Packages(c context.Context, kind string) ([]*Package, error) {
- switch kind {
- case "external", "subrepo":
- default:
- return nil, errors.New(`kind must be one of "external" or "subrepo"`)
- }
- var pkgs []*Package
- q := datastore.NewQuery("Package").Filter("Kind=", kind)
- for t := q.Run(c); ; {
- pkg := new(Package)
- _, err := t.Next(pkg)
- if err == datastore.Done {
- break
- } else if err != nil {
- return nil, err
- }
- if pkg.Path != "" {
- pkgs = append(pkgs, pkg)
- }
- }
- return pkgs, nil
-}
diff --git a/app/appengine/dash.go b/app/appengine/dash.go
index 39de722..0cfb686 100644
--- a/app/appengine/dash.go
+++ b/app/appengine/dash.go
@@ -6,35 +6,72 @@
import (
"context"
+ "crypto/tls"
+ "log"
"net/http"
+ "os"
"sort"
"strings"
+ "golang.org/x/build/maintner/maintnerd/apipb"
+ "golang.org/x/build/repos"
+ "golang.org/x/net/http2"
"google.golang.org/appengine"
+ "grpc.go4.org" // simpler, uses x/net/http2.Transport; we use this elsewhere in x/build
)
-func main() {
- // admin handlers
- handleFunc("/init", initHandler)
+var maintnerClient = createMaintnerClient()
+func main() {
// authenticated handlers
handleFunc("/building", AuthHandler(buildingHandler)) // called by coordinator during builds
handleFunc("/clear-results", AuthHandler(clearResultsHandler)) // called by x/build/cmd/retrybuilds
handleFunc("/result", AuthHandler(resultHandler)) // called by coordinator after build
- // TODO: once we use maintner for finding the git history
- // instead of having gitmirror mirror it into the dashboard,
- // then we can delete these two handlers:
- handleFunc("/commit", AuthHandler(commitHandler)) // called by gitmirror
- handleFunc("/packages", AuthHandler(packagesHandler)) // called by gitmirror
-
// public handlers
handleFunc("/", uiHandler)
handleFunc("/log/", logHandler)
+ // We used to use App Engine's static file handling support, declared in app.yaml,
+ // but it's currently broken with dev_appserver.py with the go111 runtime we use.
+ // So just do it ourselves. It doesn't buy us enough to be worth it.
+ fs := http.StripPrefix("/static", http.FileServer(http.Dir(staticDir())))
+ handleFunc("/static/", fs.ServeHTTP)
+ handleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, "https://golang.org/favicon.ico", http.StatusFound)
+ })
+
appengine.Main()
}
+func staticDir() string {
+ if pwd, _ := os.Getwd(); strings.HasSuffix(pwd, "app/appengine") {
+ return "static"
+ }
+ return "app/appengine/static"
+}
+
+func createMaintnerClient() apipb.MaintnerServiceClient {
+ addr := os.Getenv("MAINTNER_ADDR") // host[:port]
+ if addr == "" {
+ addr = "maintner.golang.org"
+ }
+ tr := &http.Transport{
+ TLSClientConfig: &tls.Config{
+ NextProtos: []string{"h2"},
+ InsecureSkipVerify: strings.HasPrefix(addr, "localhost:"),
+ },
+ }
+ hc := &http.Client{Transport: tr}
+ http2.ConfigureTransport(tr)
+
+ cc, err := grpc.NewClient(hc, "https://"+addr)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return apipb.NewMaintnerServiceClient(cc)
+}
+
func handleFunc(path string, h http.HandlerFunc) {
http.Handle(path, hstsHandler(h))
}
@@ -52,18 +89,25 @@
// (There used to be more than one dashboard, so this is now somewhat
// less important than it once was.)
type Dashboard struct {
- Name string // This dashboard's name (always "Go" nowadays)
- Namespace string // This dashboard's namespace (always "Git" nowadays)
- Packages []*Package // The project's packages to build
+ Name string // This dashboard's name (always "Go" nowadays)
+ Packages []*Package // The project's packages to build
+}
+
+// packageWithPath returns the Package in d with the provided importPath,
+// or nil if none is found.
+func (d *Dashboard) packageWithPath(importPath string) *Package {
+ for _, p := range d.Packages {
+ if p.Path == importPath {
+ return p
+ }
+ }
+ return nil
}
// Context returns a namespaced context for this dashboard, or panics if it
// fails to create a new context.
func (d *Dashboard) Context(ctx context.Context) context.Context {
- if d.Namespace == "" {
- return ctx
- }
- n, err := appengine.Namespace(ctx, d.Namespace)
+ n, err := appengine.Namespace(ctx, "Git")
if err != nil {
panic(err)
}
@@ -72,142 +116,25 @@
// goDash is the dashboard for the main go repository.
var goDash = &Dashboard{
- Name: "Go",
- Namespace: "Git",
- Packages: goPackages,
-}
-
-// goPackages is a list of all of the packages built by the main go repository.
-var goPackages = []*Package{
- {
- Kind: "go",
- Name: "Go",
- },
- {
- Kind: "subrepo",
- Name: "arch",
- Path: "golang.org/x/arch",
- },
- {
- Kind: "subrepo",
- Name: "benchmarks",
- Path: "golang.org/x/benchmarks",
- },
- {
- Kind: "subrepo",
- Name: "blog",
- Path: "golang.org/x/blog",
- },
- {
- Kind: "subrepo",
- Name: "build",
- Path: "golang.org/x/build",
- },
- {
- Kind: "subrepo",
- Name: "crypto",
- Path: "golang.org/x/crypto",
- },
- {
- Kind: "subrepo",
- Name: "debug",
- Path: "golang.org/x/debug",
- },
- {
- Kind: "subrepo",
- Name: "exp",
- Path: "golang.org/x/exp",
- },
- {
- Kind: "subrepo",
- Name: "image",
- Path: "golang.org/x/image",
- },
- {
- Kind: "subrepo",
- Name: "mobile",
- Path: "golang.org/x/mobile",
- },
- {
- Kind: "subrepo",
- Name: "net",
- Path: "golang.org/x/net",
- },
- {
- Kind: "subrepo",
- Name: "oauth2",
- Path: "golang.org/x/oauth2",
- },
- {
- Kind: "subrepo",
- Name: "perf",
- Path: "golang.org/x/perf",
- },
- {
- Kind: "subrepo",
- Name: "review",
- Path: "golang.org/x/review",
- },
- {
- Kind: "subrepo",
- Name: "sync",
- Path: "golang.org/x/sync",
- },
- {
- Kind: "subrepo",
- Name: "sys",
- Path: "golang.org/x/sys",
- },
- {
- Kind: "subrepo",
- Name: "talks",
- Path: "golang.org/x/talks",
- },
- {
- Kind: "subrepo",
- Name: "term",
- Path: "golang.org/x/term",
- },
- {
- Kind: "subrepo",
- Name: "text",
- Path: "golang.org/x/text",
- },
- {
- Kind: "subrepo",
- Name: "time",
- Path: "golang.org/x/time",
- },
- {
- Kind: "subrepo",
- Name: "tools",
- Path: "golang.org/x/tools",
- },
- {
- Kind: "subrepo",
- Name: "tour",
- Path: "golang.org/x/tour",
- },
- {
- Kind: "subrepo",
- Name: "website",
- Path: "golang.org/x/website",
+ Name: "Go",
+ Packages: []*Package{
+ {Name: "Go"},
},
}
-// supportedReleaseBranches returns a slice containing the most recent two non-security release branches
-// contained in branches.
-func supportedReleaseBranches(branches []string) (supported []string) {
- for _, b := range branches {
- if !strings.HasPrefix(b, "release-branch.go1.") ||
- len(b) != len("release-branch.go1.nn") { // assumes nn in range [10, 99]
+func init() {
+ var add []*Package
+ for _, r := range repos.ByGerritProject {
+ if r.HideFromDashboard || !strings.HasPrefix(r.ImportPath, "golang.org/x") || r.GoGerritProject == "" {
continue
}
- supported = append(supported, b)
+ add = append(add, &Package{
+ Name: r.GoGerritProject,
+ Path: r.ImportPath,
+ })
}
- sort.Strings(supported)
- if len(supported) > 2 {
- supported = supported[len(supported)-2:]
- }
- return supported
+ sort.Slice(add, func(i, j int) bool {
+ return add[i].Name < add[j].Name
+ })
+ goDash.Packages = append(goDash.Packages, add...)
}
diff --git a/app/appengine/dash_test.go b/app/appengine/dash_test.go
deleted file mode 100644
index 18a8515..0000000
--- a/app/appengine/dash_test.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "reflect"
- "testing"
-)
-
-func TestSupportedReleaseBranches(t *testing.T) {
- tests := []struct {
- in, want []string
- }{
- {
- in: []string{"master", "release-branch.go1.12", "release-branch.go1.5", "release-branch.go1.14"},
- want: []string{"release-branch.go1.12", "release-branch.go1.14"},
- },
- {
- in: []string{"master", "release-branch.go1.12", "release-branch.go1.5", "release-branch.go1.14", "release-branch.go1.15-security", "release-branch.go1.15"},
- want: []string{"release-branch.go1.14", "release-branch.go1.15"},
- },
- {
- in: []string{"master", "release-branch.go1.12", "release-branch.go1.5"},
- want: []string{"release-branch.go1.12"},
- },
- {
- in: []string{"master", "release-branch.go1.12-security"},
- want: nil,
- },
- }
- for _, tt := range tests {
- got := supportedReleaseBranches(tt.in)
- if !reflect.DeepEqual(got, tt.want) {
- t.Errorf("supportedReleaseBranches(%q) = %q; want %q", tt.in, got, tt.want)
- }
- }
-}
diff --git a/app/appengine/handler.go b/app/appengine/handler.go
index 1fb75c1..15c6ac3 100644
--- a/app/appengine/handler.go
+++ b/app/appengine/handler.go
@@ -12,14 +12,12 @@
"errors"
"fmt"
"html"
- "io/ioutil"
"net/http"
"strconv"
"strings"
"time"
"unicode/utf8"
- "golang.org/x/build/app/cache"
"golang.org/x/build/app/key"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
@@ -29,201 +27,9 @@
const (
commitsPerPage = 30
- watcherVersion = 3 // must match dashboard/watcher/watcher.go
- builderVersion = 1 // must match dashboard/builder/http.go
+ builderVersion = 1 // must match x/build/cmd/coordinator/dash.go's value
)
-// commitHandler retrieves commit data or records a new commit.
-//
-// For GET requests it returns a Commit value for the specified
-// packagePath and hash.
-//
-// For POST requests it reads a JSON-encoded Commit value from the request
-// body and creates a new Commit entity. It also updates the "tip" Tag for
-// each new commit at tip.
-//
-// This handler is used by a gobuilder process in -commit mode.
-func commitHandler(r *http.Request) (interface{}, error) {
- c := contextForRequest(r)
- com := new(Commit)
-
- if r.Method == "GET" {
- com.PackagePath = r.FormValue("packagePath")
- com.Hash = r.FormValue("hash")
- err := datastore.Get(c, com.Key(c), com)
- if com.Num == 0 && com.Desc == "" {
- // Perf builder might have written an incomplete Commit.
- // Pretend it doesn't exist, so that we can get complete details.
- err = datastore.ErrNoSuchEntity
- }
- if err != nil {
- if err == datastore.ErrNoSuchEntity {
- // This error string is special.
- // The commit watcher expects it.
- // Do not change it.
- return nil, errors.New("Commit not found")
- }
- return nil, fmt.Errorf("getting Commit: %v", err)
- }
- if com.Num == 0 {
- // Corrupt state which shouldn't happen but does.
- // Return an error so builders' commit loops will
- // be willing to retry submitting this commit.
- return nil, errors.New("in datastore with zero Num")
- }
- if com.Desc == "" || com.User == "" {
- // Also shouldn't happen, but at least happened
- // once on a single commit when trying to fix data
- // in the datastore viewer UI?
- return nil, errors.New("missing field")
- }
- // Strip potentially large and unnecessary fields.
- com.ResultData = nil
- return com, nil
- }
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
- if !isMasterKey(c, r.FormValue("key")) {
- return nil, errors.New("can only POST commits with master key")
- }
-
- v, _ := strconv.Atoi(r.FormValue("version"))
- if v != watcherVersion {
- return nil, fmt.Errorf("rejecting POST from commit watcher; need version %v instead of %v",
- watcherVersion, v)
- }
-
- // POST request
- body, err := ioutil.ReadAll(r.Body)
- r.Body.Close()
- if err != nil {
- return nil, fmt.Errorf("reading Body: %v", err)
- }
- if err := json.Unmarshal(body, com); err != nil {
- return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
- }
- com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
- if err := com.Valid(); err != nil {
- return nil, fmt.Errorf("validating Commit: %v", err)
- }
- defer cache.Tick(c)
- tx := func(c context.Context) error {
- return addCommit(c, com)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-// addCommit adds the Commit entity to the datastore and updates the tip Tag.
-// It must be run inside a datastore transaction.
-func addCommit(c context.Context, com *Commit) error {
- var ec Commit // existing commit
- isUpdate := false
- err := datastore.Get(c, com.Key(c), &ec)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("getting Commit: %v", err)
- }
- if err == nil {
- // Commit already in the datastore. Any fields different?
- // If not, don't do anything.
- changes := (com.Num != 0 && com.Num != ec.Num) ||
- com.ParentHash != ec.ParentHash ||
- com.Desc != ec.Desc ||
- com.User != ec.User ||
- !com.Time.Equal(ec.Time)
- if !changes {
- return nil
- }
- ec.ParentHash = com.ParentHash
- ec.Desc = com.Desc
- ec.User = com.User
- if !com.Time.IsZero() {
- ec.Time = com.Time
- }
- if com.Num != 0 {
- ec.Num = com.Num
- }
- isUpdate = true
- com = &ec
- }
- p, err := GetPackage(c, com.PackagePath)
- if err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- if com.Num == 0 {
- // get the next commit number
- com.Num = p.NextNum
- p.NextNum++
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- return fmt.Errorf("putting Package: %v", err)
- }
- } else if com.Num >= p.NextNum {
- p.NextNum = com.Num + 1
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- return fmt.Errorf("putting Package: %v", err)
- }
- }
- // if this isn't the first Commit test the parent commit exists.
- // The all zeros are returned by hg's p1node template for parentless commits.
- if com.ParentHash != "" && com.ParentHash != "0000000000000000000000000000000000000000" && com.ParentHash != "0000" {
- n, err := datastore.NewQuery("Commit").
- Filter("Hash =", com.ParentHash).
- Ancestor(p.Key(c)).
- Count(c)
- err = filterDatastoreError(err)
- if err != nil {
- return fmt.Errorf("testing for parent Commit: %v", err)
- }
- if n == 0 {
- return errors.New("parent commit not found")
- }
- } else if com.Num != 1 {
- // This is the first commit; fail if it is not number 1.
- // (This will happen if we try to upload a new/different repo
- // where there is already commit data. A bad thing to do.)
- return errors.New("this package already has a first commit; aborting")
- }
- // Update the relevant Tag entity, if applicable.
- if !isUpdate && p.Path == "" {
- var t *Tag
- if com.Branch == "master" {
- t = &Tag{Kind: "tip", Hash: com.Hash}
- }
- if strings.HasPrefix(com.Branch, "release-branch.") {
- t = &Tag{Kind: "release", Name: com.Branch, Hash: com.Hash}
- }
- if t != nil {
- if _, err = datastore.Put(c, t.Key(c), t); err != nil {
- return fmt.Errorf("putting Tag: %v", err)
- }
- }
- }
- // put the Commit
- if err = putCommit(c, com); err != nil {
- return err
- }
- return nil
-}
-
-// packagesHandler returns a list of the non-Go Packages monitored
-// by the dashboard.
-func packagesHandler(r *http.Request) (interface{}, error) {
- kind := r.FormValue("kind")
- c := contextForRequest(r)
- now := cache.Now(c)
- key := "build-packages-" + kind
- var p []*Package
- if cache.Get(c, r, now, key, &p) {
- return p, nil
- }
- p, err := Packages(c, kind)
- if err != nil {
- return nil, err
- }
- cache.Set(c, r, now, key, p)
- return p, nil
-}
-
// buildingHandler records that a build is in progress.
// The data is only stored in memcache and with a timeout. It's assumed
// that the build system will periodically refresh this if the build
@@ -249,7 +55,7 @@
// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
-// creates a new Result entity, and updates the relevant Commit entity.
+// creates a new Result entity, and creates or updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, error) {
@@ -272,7 +78,6 @@
if err := res.Valid(); err != nil {
return nil, fmt.Errorf("validating Result: %v", err)
}
- defer cache.Tick(c)
// store the Log text if supplied
if len(res.Log) > 0 {
hash, err := PutLog(c, res.Log)
@@ -282,8 +87,7 @@
res.LogHash = hash
}
tx := func(c context.Context) error {
- // check Package exists
- if _, err := GetPackage(c, res.PackagePath); err != nil {
+ if _, err := getOrMakePackageInTx(c, res.PackagePath); err != nil {
return fmt.Errorf("GetPackage: %v", err)
}
// put Result
@@ -329,50 +133,49 @@
w.Write(b)
}
-// clearResultsHandler purges the last commitsPerPage results for the given builder.
-// It optionally takes a comma-separated list of specific hashes to clear.
+// clearResultsHandler purge a single build failure from the dashboard.
+// It currently only supports the main Go repo.
func clearResultsHandler(r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
builder := r.FormValue("builder")
+ hash := r.FormValue("hash")
if builder == "" {
- return nil, errors.New("must specify a builder")
+ return nil, errors.New("missing 'builder'")
}
- clearAll := r.FormValue("hash") == ""
- hash := strings.Split(r.FormValue("hash"), ",")
+ if hash == "" {
+ return nil, errors.New("missing 'hash'")
+ }
- c := contextForRequest(r)
- defer cache.Tick(c)
- pkg := (&Package{}).Key(c) // TODO(adg): support clearing sub-repos
- err := datastore.RunInTransaction(c, func(c context.Context) error {
- var coms []*Commit
- keys, err := datastore.NewQuery("Commit").
- Ancestor(pkg).
- Order("-Num").
- Limit(commitsPerPage).
- GetAll(c, &coms)
+ ctx := contextForRequest(r)
+
+ err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
+ c := &Commit{
+ PackagePath: "", // TODO(adg): support clearing sub-repos
+ Hash: hash,
+ }
+ err := datastore.Get(ctx, c.Key(ctx), c)
err = filterDatastoreError(err)
+ if err == datastore.ErrNoSuchEntity {
+ // Doesn't exist, so no build to clear.
+ return nil
+ }
if err != nil {
return err
}
- var rKeys []*datastore.Key
- for _, com := range coms {
- if !(clearAll || contains(hash, com.Hash)) {
- continue
- }
- r := com.Result(builder, "")
- if r == nil {
- continue
- }
- com.RemoveResult(r)
- rKeys = append(rKeys, r.Key(c))
+
+ r := c.Result(builder, "")
+ if r == nil {
+ // No result, so nothing to clear.
+ return nil
}
- _, err = datastore.PutMulti(c, keys, coms)
+ c.RemoveResult(r)
+ _, err = datastore.Put(ctx, c.Key(ctx), c)
if err != nil {
return err
}
- return datastore.DeleteMulti(c, rKeys)
+ return datastore.Delete(ctx, r.Key(ctx))
}, nil)
return nil, err
}
diff --git a/app/appengine/init.go b/app/appengine/init.go
deleted file mode 100644
index 1be69fc..0000000
--- a/app/appengine/init.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "fmt"
- "net/http"
-
- "google.golang.org/appengine"
- "google.golang.org/appengine/datastore"
-
- "golang.org/x/build/app/cache"
- "golang.org/x/build/app/key"
-)
-
-func initHandler(w http.ResponseWriter, r *http.Request) {
- d := goDash
- c := d.Context(appengine.NewContext(r))
- defer cache.Tick(c)
- for _, p := range d.Packages {
- err := datastore.Get(c, p.Key(c), new(Package))
- if _, ok := err.(*datastore.ErrFieldMismatch); ok {
- // Some fields have been removed, so it's okay to ignore this error.
- err = nil
- }
- if err == nil {
- continue
- } else if err != datastore.ErrNoSuchEntity {
- logErr(w, r, err)
- return
- }
- p.NextNum = 1 // So we can add the first commit.
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- logErr(w, r, err)
- return
- }
- }
-
- // Create secret key.
- key.Secret(c)
-
- fmt.Fprint(w, "OK")
-}
diff --git a/app/appengine/ui.go b/app/appengine/ui.go
index 725c8b1..e6319b2 100644
--- a/app/appengine/ui.go
+++ b/app/appengine/ui.go
@@ -8,6 +8,7 @@
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"html/template"
"net/http"
@@ -18,9 +19,12 @@
"strings"
"time"
- "golang.org/x/build/app/cache"
"golang.org/x/build/dashboard"
+ "golang.org/x/build/maintner/maintnerd/apipb"
+ "golang.org/x/build/repos"
"golang.org/x/build/types"
+ "grpc.go4.org"
+ "grpc.go4.org/codes"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
@@ -28,193 +32,371 @@
"google.golang.org/appengine/memcache"
)
-// uiHandler draws the build status page.
+// uiHandler is the HTTP handler for the https://build.golang.org/.
func uiHandler(w http.ResponseWriter, r *http.Request) {
- d := goDash
- c := d.Context(appengine.NewContext(r))
- now := cache.Now(c)
- key := "build-ui"
-
- mode := r.FormValue("mode")
-
- page, _ := strconv.Atoi(r.FormValue("page"))
- if page < 0 {
- page = 0
- }
- key += fmt.Sprintf("-page%v", page)
-
- repo := r.FormValue("repo")
- if repo != "" {
- key += "-repo-" + repo
+ view, err := viewForRequest(r)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
}
- branch := r.FormValue("branch")
- switch branch {
- case "all":
- branch = ""
+ dashReq, err := dashboardRequest(view, r)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ // TODO: if we end up fetching the building state from the
+ // coordinator too, do that concurrently in an
+ // x/sync/errgroup.Group here. But for now we're only doing
+ // one RPC call.
+ ctx := goDash.Context(appengine.NewContext(r))
+ dashRes, err := maintnerClient.GetDashboard(ctx, dashReq)
+ if err != nil {
+ http.Error(w, "maintner.GetDashboard: "+err.Error(), httpStatusOfErr(err))
+ return
+ }
+
+ tb := newUITemplateDataBuilder(view, dashReq, dashRes)
+ data, err := tb.buildTemplateData(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ view.ServeDashboard(w, r, data)
+}
+
+// dashboardView is something that can render uiTemplateData.
+// See viewForRequest.
+type dashboardView interface {
+ ServeDashboard(w http.ResponseWriter, r *http.Request, data *uiTemplateData)
+}
+
+// viewForRequest selects the dashboardView based on the HTTP
+// request's "mode" parameter. Any error should be considered
+// an HTTP 400 Bad Request.
+func viewForRequest(r *http.Request) (dashboardView, error) {
+ if r.Method != "GET" && r.Method != "HEAD" {
+ return nil, errors.New("unsupported method")
+ }
+ switch r.FormValue("mode") {
+ case "failures":
+ return failuresView{}, nil
+ case "json":
+ return jsonView{}, nil
case "":
- branch = "master"
+ return htmlView{}, nil
}
- if repo != "" || mode == "json" {
- // Don't filter on branches in sub-repos.
- // TODO(adg): figure out how to make this work sensibly.
- // Don't filter on branches in json mode.
- branch = ""
+ return nil, errors.New("unsupported mode argument")
+}
+
+type commitInPackage struct {
+ packagePath string // "" for Go, else package import path
+ commit string // git commit hash
+}
+
+// uiTemplateDataBuilder builds the uiTemplateData used by the various
+// dashboardViews. That is, it maps the maintner protobuf response to
+// the data structure needed by the dashboardView/template.
+type uiTemplateDataBuilder struct {
+ view dashboardView
+ req *apipb.DashboardRequest
+ res *apipb.DashboardResponse
+
+ // testCommitData, if non-nil, provides an alternate data
+ // source to use for testing instead of making real datastore
+ // calls. The keys are stringified datastore.Keys.
+ testCommitData map[string]*Commit
+}
+
+// newUITemplateDataBuilder returns a new uiTemplateDataBuilder for a
+// given view, dashboard request, and dashboard response from
+// maintner.
+func newUITemplateDataBuilder(view dashboardView, req *apipb.DashboardRequest, res *apipb.DashboardResponse) *uiTemplateDataBuilder {
+ return &uiTemplateDataBuilder{
+ view: view,
+ req: req,
+ res: res,
}
- if branch != "" {
- key += "-branch-" + branch
+}
+
+// getCommitsToLoad returns a set (all values are true) of which commits to load from
+// the datastore.
+func (tb *uiTemplateDataBuilder) getCommitsToLoad() map[commitInPackage]bool {
+ m := make(map[commitInPackage]bool)
+ add := func(packagePath, commit string) {
+ m[commitInPackage{packagePath: packagePath, commit: commit}] = true
}
- hashes := r.Form["hash"]
-
- var data uiTemplateData
- if len(hashes) > 0 || !cache.Get(c, r, now, key, &data) {
-
- pkg := &Package{} // empty package is the main repository
- if repo != "" {
- var err error
- pkg, err = GetPackage(c, repo)
- if err != nil {
- logErr(w, r, err)
- return
+ for _, dc := range tb.res.Commits {
+ add(tb.req.Repo, dc.Commit)
+ }
+ // We also want to load the Commits for the x/repo heads.
+ if tb.showXRepoSection() {
+ for _, rh := range tb.res.RepoHeads {
+ if path := repoImportPath(rh); path != "" {
+ add(path, rh.Commit.Commit)
}
}
- var commits []*Commit
- var err error
- if len(hashes) > 0 {
- commits, err = fetchCommits(c, pkg, hashes)
- } else {
- commits, err = dashCommits(c, pkg, page, branch)
- }
- if err != nil {
- logErr(w, r, err)
- return
- }
- branches := listBranches(c)
- releaseBranches := supportedReleaseBranches(branches)
- builders := commitBuilders(commits, releaseBranches)
+ }
+ return m
+}
- var tagState []*TagState
- // Only show sub-repo state on first page of normal repo view.
- if pkg.Kind == "" && len(hashes) == 0 && page == 0 && (branch == "" || branch == "master") {
- s, err := GetTagState(c, "tip", "")
- if err != nil {
- if err == datastore.ErrNoSuchEntity {
- if appengine.IsDevAppServer() {
- goto BuildData
- }
- err = fmt.Errorf("tip tag not found")
- }
- logErr(w, r, err)
- return
+// loadDatastoreCommits loads the commits given in the keys of the
+// want map. The returned map is keyed by the git hash and may not
+// contain items that didn't exist in the datastore. (It is not an
+// error if 1 or all don't exist.)
+func (tb *uiTemplateDataBuilder) loadDatastoreCommits(ctx context.Context, want map[commitInPackage]bool) (map[string]*Commit, error) {
+ ret := map[string]*Commit{}
+
+ // Allow tests to fake what the datastore would've loaded, and
+ // thus also allow tests to be run without a real (or
+ // dev_appserver-based fake) datastore.
+ if m := tb.testCommitData; m != nil {
+ for k := range want {
+ if c, ok := m[k.commit]; ok {
+ ret[k.commit] = c
}
- tagState = []*TagState{s}
- for _, b := range releaseBranches {
- s, err := GetTagState(c, "release", b)
- if err == datastore.ErrNoSuchEntity {
+ }
+ return ret, nil
+ }
+
+ var keys []*datastore.Key
+ for k := range want {
+ key := (&Commit{
+ PackagePath: k.packagePath,
+ Hash: k.commit,
+ }).Key(ctx)
+ keys = append(keys, key)
+ }
+ commits, err := fetchCommits(ctx, keys)
+ if err != nil {
+ return nil, fmt.Errorf("fetchCommits: %v", err)
+ }
+ for _, c := range commits {
+ ret[c.Hash] = c
+ }
+ return ret, nil
+}
+
+// formatGitAuthor formats the git author name and email (as split by
+// maintner) back into the unified string how they're stored in a git
+// commit, so the shortUser func (used by the HTML template) can parse
+// back out the email part's username later. Maybe we could plumb down
+// the parsed proto into the template later.
+func formatGitAuthor(name, email string) string {
+ name = strings.TrimSpace(name)
+ email = strings.TrimSpace(email)
+ if name != "" && email != "" {
+ return fmt.Sprintf("%s <%s>", name, email)
+ }
+ if name != "" {
+ return name
+ }
+ return "<" + email + ">"
+}
+
+// newCommitInfo returns a new CommitInfo populated for the template
+// data given a repo name and a dashboard commit from that repo, using
+// previously loaded datastore commit info in tb.
+func (tb *uiTemplateDataBuilder) newCommitInfo(dsCommits map[string]*Commit, repo string, dc *apipb.DashCommit) *CommitInfo {
+ ci := &CommitInfo{
+ Hash: dc.Commit,
+ PackagePath: repo,
+ User: formatGitAuthor(dc.AuthorName, dc.AuthorEmail),
+ Desc: cleanTitle(dc.Title, tb.req.Branch),
+ Time: time.Unix(dc.CommitTimeSec, 0),
+ }
+ if dsc, ok := dsCommits[dc.Commit]; ok {
+ ci.ResultData = dsc.ResultData
+ }
+ // For non-go repos, add the rows for the Go commits that were
+ // at HEAD overlapping in time with dc.Commit.
+ isGo := tb.req.Repo == "" || tb.req.Repo == "go"
+ if !isGo {
+ if dc.GoCommitAtTime != "" {
+ ci.addEmptyResultGoHash(dc.GoCommitAtTime)
+ }
+ if dc.GoCommitLatest != "" && dc.GoCommitLatest != dc.GoCommitAtTime {
+ ci.addEmptyResultGoHash(dc.GoCommitLatest)
+ }
+ }
+ return ci
+}
+
+// showXRepoSection reports whether the dashboard should show the state of the x/foo repos at the bottom of
+// the page in the three branches (master, latest release branch, two releases ago).
+func (tb *uiTemplateDataBuilder) showXRepoSection() bool {
+ return tb.req.Page == 0 &&
+ (tb.req.Branch == "" || tb.req.Branch == "master" || tb.req.Branch == "mixed") &&
+ (tb.req.Repo == "" || tb.req.Repo == "go")
+}
+
+// repoImportPath returns the import path for rh, unless rh is the
+// main "go" repo or is configured to be hidden from the dashboard, in
+// which case it returns the empty string.
+func repoImportPath(rh *apipb.DashRepoHead) string {
+ if rh.GerritProject == "go" {
+ return ""
+ }
+ ri, ok := repos.ByGerritProject[rh.GerritProject]
+ if !ok || ri.HideFromDashboard {
+ return ""
+ }
+ return ri.ImportPath
+}
+
+func (tb *uiTemplateDataBuilder) buildTemplateData(ctx context.Context) (*uiTemplateData, error) {
+ dsCommits, err := tb.loadDatastoreCommits(ctx, tb.getCommitsToLoad())
+ if err != nil {
+ return nil, err
+ }
+
+ var commits []*CommitInfo
+ for _, dc := range tb.res.Commits {
+ ci := tb.newCommitInfo(dsCommits, tb.req.Repo, dc)
+ commits = append(commits, ci)
+ }
+
+ // x/ repo sections at bottom (each is a "TagState", for historical reasons)
+ var xRepoSections []*TagState
+ if tb.showXRepoSection() {
+ for _, gorel := range tb.res.Releases {
+ ts := &TagState{
+ Name: gorel.BranchName,
+ Tag: &CommitInfo{ // only a minimally populated version is needed by the template
+ Hash: gorel.BranchCommit,
+ },
+ }
+ for _, rh := range tb.res.RepoHeads {
+ path := repoImportPath(rh)
+ if path == "" {
continue
}
- err = filterDatastoreError(err)
- if err != nil {
- logErr(w, r, err)
- return
- }
- tagState = append(tagState, s)
+ ts.Packages = append(ts.Packages, &PackageState{
+ Package: &Package{
+ Name: rh.GerritProject,
+ Path: path,
+ },
+ Commit: tb.newCommitInfo(dsCommits, path, rh.Commit),
+ })
}
- }
- // Sort tagState in reverse lexical order by name so higher
- // numbered release branches show first for subrepos
- // https://build.golang.org/ after master. We want the subrepo
- // order to be "master, release-branch.go1.12,
- // release-branch.go1.11" so they're in order by date (newest
- // first). If we weren't already at two digit minor versions we'd
- // need to parse the branch name, but we can be lazy now
- // and just do a string compare.
- sort.Slice(tagState, func(i, j int) bool { // is item 'i' less than item 'j'?
- ni, nj := tagState[i].Name, tagState[j].Name
- switch {
- case ni == "master":
- return true // an i of "master" is always first
- case nj == "master":
- return false // if i wasn't "master", it can't be less than j's "master"
- default:
- return ni > nj // "release-branch.go1.12" > "release-branch.go1.11", so 1.12 sorts earlier
- }
- })
-
- BuildData:
- p := &Pagination{}
- if len(commits) == commitsPerPage {
- p.Next = page + 1
- }
- if page > 0 {
- p.Prev = page - 1
- p.HasPrev = true
- }
-
- data = uiTemplateData{
- Package: pkg,
- Commits: commits,
- Builders: builders,
- TagState: tagState,
- Pagination: p,
- Branches: branches,
- Branch: branch,
- }
- if len(hashes) == 0 {
- cache.Set(c, r, now, key, &data)
+ sort.Slice(ts.Packages, func(i, j int) bool {
+ return ts.Packages[i].Package.Name < ts.Packages[j].Package.Name
+ })
+ xRepoSections = append(xRepoSections, ts)
}
}
- data.Dashboard = d
- switch mode {
- case "failures":
- failuresHandler(w, r, &data)
- return
- case "json":
- jsonHandler(w, r, &data)
- return
+ // Release Branches
+ var releaseBranches []string
+ for _, gr := range tb.res.Releases {
+ if gr.BranchName != "master" {
+ releaseBranches = append(releaseBranches, gr.BranchName)
+ }
}
- // In the UI, when viewing the master branch of the main
- // Go repository, hide builders that aren't active on it.
- if repo == "" && branch == "master" {
- data.Builders = onlyGoMasterBuilders(data.Builders)
+ builders := commitBuilders(commits, releaseBranches)
+ data := &uiTemplateData{
+ Dashboard: goDash,
+ Package: goDash.packageWithPath(tb.req.Repo),
+ Commits: commits,
+ Builders: builders,
+ TagState: xRepoSections,
+ Pagination: &Pagination{},
+ Branches: tb.res.Branches,
+ Branch: tb.req.Branch,
}
- // Populate building URLs for the HTML UI only.
- data.populateBuildingURLs(c)
+ if tb.res.CommitsTruncated {
+ data.Pagination.Next = int(tb.req.Page) + 1
+ }
+ if tb.req.Page > 0 {
+ data.Pagination.Prev = int(tb.req.Page) - 1
+ data.Pagination.HasPrev = true
+ }
+ if tb.view == (htmlView{}) {
+ // In the UI, when viewing the master branch of the main
+ // Go repository, hide builders that aren't active on it.
+ if tb.req.Repo == "" && tb.req.Branch == "master" {
+ data.Builders = onlyGoMasterBuilders(data.Builders)
+ }
+
+ // Populate building URLs for the HTML UI only.
+ data.populateBuildingURLs(ctx)
+ }
+
+ return data, nil
+}
+
+// htmlView renders the HTML (default) form of https://build.golang.org/ with no mode parameter.
+type htmlView struct{}
+
+func (htmlView) ServeDashboard(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
var buf bytes.Buffer
- if err := uiTemplate.Execute(&buf, &data); err != nil {
+ if err := uiTemplate.Execute(&buf, data); err != nil {
logErr(w, r, err)
return
}
buf.WriteTo(w)
}
-func listBranches(c context.Context) (branches []string) {
- var commits []*Commit
- _, err := datastore.NewQuery("Commit").Distinct().Project("Branch").GetAll(c, &commits)
- err = filterDatastoreError(err)
- if err != nil {
- log.Errorf(c, "listBranches: %v", err)
- return
- }
- for _, c := range commits {
- if strings.HasPrefix(c.Branch, "release-branch.go") &&
- strings.HasSuffix(c.Branch, "-security") {
- continue
+// dashboardRequest is a pure function that maps the provided HTTP
+// request to a maintner DashboardRequest and lightly validates the
+// HTTP request for the root dashboard handler. (It does not validate
+// that, say, branches or repos are valid.)
+// Any returned error is an HTTP 400 Bad Request.
+func dashboardRequest(view dashboardView, r *http.Request) (*apipb.DashboardRequest, error) {
+ page := 0
+ if s := r.FormValue("page"); s != "" {
+ var err error
+ page, err = strconv.Atoi(r.FormValue("page"))
+ if err != nil {
+ return nil, fmt.Errorf("invalid page value %q", s)
}
- branches = append(branches, c.Branch)
+ if page < 0 {
+ return nil, errors.New("negative page")
+ }
}
- return
+
+ repo := r.FormValue("repo") // empty for main go repo, else e.g. "golang.org/x/net"
+
+ branch := r.FormValue("branch")
+ if branch == "" {
+ branch = "master"
+ }
+ return &apipb.DashboardRequest{
+ Page: int32(page),
+ Branch: branch,
+ Repo: repo,
+ MaxCommits: commitsPerPage,
+ }, nil
}
-// failuresHandler is https://build.golang.org/?mode=failures, where it outputs
+// cleanTitle returns a cleaned version of the provided title for
+// users viewing the provided viewBranch.
+func cleanTitle(title, viewBranch string) string {
+ // Don't rewrite anything for master and mixed.
+ if viewBranch == "master" || viewBranch == "mixed" {
+ return title
+ }
+ // Strip the "[release-branch.go1.n]" prefixes from commit messages
+ // when looking at a branch.
+ if strings.HasPrefix(title, "[") {
+ if i := strings.IndexByte(title, ']'); i != -1 {
+ return strings.TrimSpace(title[i+1:])
+ }
+ }
+ return title
+}
+
+// failuresView renders https://build.golang.org/?mode=failures, where it outputs
// one line per failure on the front page, in the form:
// hash builder failure-url
-func failuresHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
+type failuresView struct{}
+
+func (failuresView) ServeDashboard(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
w.Header().Set("Content-Type", "text/plain")
for _, c := range data.Commits {
for _, b := range data.Builders {
@@ -228,9 +410,11 @@
}
}
-// jsonHandler is https://build.golang.org/?mode=json
+// jsonView renders https://build.golang.org/?mode=json.
// The output is a types.BuildStatus JSON object.
-func jsonHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
+type jsonView struct{}
+
+func (jsonView) ServeDashboard(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
// cell returns one of "" (no data), "ok", or a failure URL.
cell := func(res *Result) string {
switch {
@@ -290,12 +474,9 @@
// commitToBuildRevision fills in the fields of BuildRevision rev that
// are derived from Commit c.
-func commitToBuildRevision(c *Commit, rev *types.BuildRevision) {
+func commitToBuildRevision(c *CommitInfo, rev *types.BuildRevision) {
rev.Revision = c.Hash
- // TODO: A comment may have more than one parent.
- rev.ParentRevisions = []string{c.ParentHash}
rev.Date = c.Time.Format(time.RFC3339)
- rev.Branch = c.Branch
rev.Author = c.User
rev.Desc = c.Desc
}
@@ -305,63 +486,43 @@
HasPrev bool
}
-// dashCommits gets a slice of the latest Commits to the current dashboard.
-// If page > 0 it paginates by commitsPerPage.
-func dashCommits(c context.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
- offset := page * commitsPerPage
- q := datastore.NewQuery("Commit").
- Ancestor(pkg.Key(c)).
- Order("-Num")
-
- if branch != "" {
- q = q.Filter("Branch =", branch)
+// fetchCommits loads any commits that exist given by keys.
+// It is not an error if a commit doesn't exist.
+// Only commits that were found in datastore are returned,
+// in an unspecified order.
+func fetchCommits(ctx context.Context, keys []*datastore.Key) ([]*Commit, error) {
+ if len(keys) == 0 {
+ return nil, nil
+ }
+ out := make([]*Commit, len(keys))
+ for i := range keys {
+ out[i] = new(Commit)
}
- var commits []*Commit
- _, err := q.Limit(commitsPerPage).Offset(offset).
- GetAll(c, &commits)
+ err := datastore.GetMulti(ctx, keys, out)
err = filterDatastoreError(err)
-
- // If we're running locally and don't have data, return some test data.
- // This lets people hack on the UI without setting up gitmirror & friends.
- if len(commits) == 0 && appengine.IsDevAppServer() && err == nil {
- commits = []*Commit{
- {
- Hash: "7d7c6a97f815e9279d08cfaea7d5efb5e90695a8",
- ParentHash: "",
- Num: 1,
- User: "bwk",
- Desc: "hello, world",
- },
+ err = filterNoSuchEntity(err)
+ if err != nil {
+ return nil, err
+ }
+ filtered := out[:0]
+ for _, c := range out {
+ if c.Valid() { // that is, successfully loaded
+ filtered = append(filtered, c)
}
}
- return commits, err
-}
-
-// fetchCommits gets a slice of the specific commit hashes
-func fetchCommits(c context.Context, pkg *Package, hashes []string) ([]*Commit, error) {
- var out []*Commit
- var keys []*datastore.Key
- for _, hash := range hashes {
- commit := &Commit{
- Hash: hash,
- PackagePath: pkg.Path,
- }
- out = append(out, commit)
- keys = append(keys, commit.Key(c))
- }
- err := datastore.GetMulti(c, keys, out)
- err = filterDatastoreError(err)
- return out, err
+ return filtered, nil
}
// commitBuilders returns the names of active builders that provided
// Results for the provided commits.
-func commitBuilders(commits []*Commit, releaseBranches []string) []string {
+func commitBuilders(commits []*CommitInfo, releaseBranches []string) []string {
builders := make(map[string]bool)
for _, commit := range commits {
for _, r := range commit.Results() {
- builders[r.Builder] = true
+ if r.Builder != "" {
+ builders[r.Builder] = true
+ }
}
}
// Add all known builders from the builder configuration too.
@@ -475,10 +636,10 @@
"dragonfly": 6,
}
-// TagState represents the state of all Packages at a Tag.
+// TagState represents the state of all Packages at a branch.
type TagState struct {
- Name string // "tip", "release-branch.go1.4", etc
- Tag *Commit
+ Name string // Go branch name: "master", "release-branch.go1.4", etc
+ Tag *CommitInfo // current Go commit on the Name branch
Packages []*PackageState
}
@@ -491,43 +652,49 @@
return ts.Name
}
-// PackageState represents the state of a Package at a Tag.
+// PackageState represents the state of a Package (x/foo repo) for given Go branch.
type PackageState struct {
Package *Package
- Commit *Commit
+ Commit *CommitInfo
}
-// GetTagState fetches the results for all Go subrepos at the specified Tag.
-// (Kind is "tip" or "release"; name is like "release-branch.go1.4".)
-func GetTagState(c context.Context, kind, name string) (*TagState, error) {
- tag, err := GetTag(c, kind, name)
- if err != nil {
- return nil, err
- }
- pkgs, err := Packages(c, "subrepo")
- if err != nil {
- return nil, err
- }
- st := TagState{Name: tag.String()}
- for _, pkg := range pkgs {
- com, err := pkg.LastCommit(c)
- if err != nil {
- log.Warningf(c, "%v: no Commit found: %v", pkg, err)
- continue
+// A CommitInfo is a struct for use by html/template package.
+// It is not stored in the datastore.
+type CommitInfo struct {
+ Hash string
+
+ // ResultData is a copy of the Commit.ResultData field from datastore.
+ ResultData []string
+
+ buildingURLs map[builderAndGoHash]string
+
+ PackagePath string // (empty for main repo commits)
+ User string // "Foo Bar <foo@bar.com>"
+ Desc string // git commit title
+ Time time.Time // commit time
+}
+
+// addEmptyResultGoHash adds an empty result containing goHash to
+// ci.ResultData, unless ci already contains a result for that hash.
+// This is used for non-go repos to show the go commits (both earliest
+// and latest) that correspond to this repo's commit time. We add an
+// empty result so it shows up on the dashboard (both for humans, and
+// in JSON form for the coordinator to pick up as work). Once the
+// coordinator does that work and posts its result, then ResultData
+// will be populate and this turns into a no-op.
+func (ci *CommitInfo) addEmptyResultGoHash(goHash string) {
+ for _, exist := range ci.ResultData {
+ if strings.Contains(exist, goHash) {
+ return
}
- st.Packages = append(st.Packages, &PackageState{pkg, com})
}
- st.Tag, err = tag.Commit(c)
- if err != nil {
- return nil, err
- }
- return &st, nil
+ ci.ResultData = append(ci.ResultData, (&Result{GoHash: goHash}).Data())
}
type uiTemplateData struct {
Dashboard *Dashboard
Package *Package
- Commits []*Commit
+ Commits []*CommitInfo
Builders []string
TagState []*TagState
Pagination *Pagination
@@ -541,15 +708,22 @@
return fmt.Sprintf("building|%v|%v|%v", hash, goHash, builder)
}
+// skipMemcacheForTest, if true, disables memcache operations for use in tests.
+var skipMemcacheForTest = false
+
// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
// URLs of builds which are currently in progress.
func (td *uiTemplateData) populateBuildingURLs(ctx context.Context) {
+ if skipMemcacheForTest {
+ return
+ }
+
// need are memcache keys: "building|<hash>|<gohash>|<builder>"
// The hash is of the main "go" repo, or the subrepo commit hash.
// The gohash is empty for the main repo, else it's the Go hash.
var need []string
- commit := map[string]*Commit{} // commit hash -> Commit
+ commit := map[string]*CommitInfo{} // commit hash -> Commit
// Gather pending commits for main repo.
for _, b := range td.Builders {
@@ -619,6 +793,14 @@
"tail": tail,
"unsupported": unsupported,
"isUntested": isUntested,
+ "formatTime": formatTime,
+}
+
+func formatTime(t time.Time) string {
+ if t.Year() != time.Now().Year() {
+ return t.Format("02 Jan 06")
+ }
+ return t.Format("02 Jan 15:04")
}
func splitDash(s string) (string, string) {
@@ -742,3 +924,15 @@
}
return filepath.Join("app/appengine", base)
}
+
+func httpStatusOfErr(err error) int {
+ fmt.Fprintf(os.Stderr, "Got error: %#v, code %v\n", err, grpc.Code(err))
+ switch grpc.Code(err) {
+ case codes.NotFound:
+ return http.StatusNotFound
+ case codes.InvalidArgument:
+ return http.StatusBadRequest
+ default:
+ return http.StatusInternalServerError
+ }
+}
diff --git a/app/appengine/ui.html b/app/appengine/ui.html
index c19bbc0..a7ee7f1 100644
--- a/app/appengine/ui.html
+++ b/app/appengine/ui.html
@@ -114,7 +114,7 @@
<td class="hash"><a href="https://go-review.googlesource.com/q/{{$h}}">{{shortHash $h}}</a></td>
{{end}}
<td class="user" title="{{$c.User}}">{{shortUser $c.User}}</td>
- <td class="time">{{$c.Time.Format "02 Jan 15:04"}}</td>
+ <td class="time">{{formatTime $c.Time}}</td>
<td class="desc" title="{{$c.Desc}}">{{shortDesc $c.Desc}}</td>
{{end}}
{{range $builderName := $.Builders}}
@@ -211,7 +211,7 @@
</td>
{{with $pkg.Commit}}
<td class="user" title="{{.User}}">{{shortUser .User}}</td>
- <td class="time">{{.Time.Format "02 Jan 15:04"}}</td>
+ <td class="time">{{formatTime .Time}}</td>
<td class="desc" title="{{.Desc}}">{{shortDesc .Desc}}</td>
{{end}}
{{range $builderName := $.Builders}}
diff --git a/app/appengine/ui_test.go b/app/appengine/ui_test.go
new file mode 100644
index 0000000..e8d5cb1
--- /dev/null
+++ b/app/appengine/ui_test.go
@@ -0,0 +1,320 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "golang.org/x/build/dashboard"
+ "golang.org/x/build/maintner/maintnerd/apipb"
+)
+
+func TestUITemplateDataBuilder(t *testing.T) {
+ // Thin the list of builders to make this test's data lighter
+ // and require less maintenance keeping it in sync.
+ // Also disable use of memcache.
+ origBuilders := dashboard.Builders
+ defer func() {
+ dashboard.Builders = origBuilders
+ skipMemcacheForTest = false
+ }()
+ dashboard.Builders = map[string]*dashboard.BuildConfig{
+ "linux-amd64": origBuilders["linux-amd64"],
+ "linux-386": origBuilders["linux-386"],
+ }
+ skipMemcacheForTest = true
+
+ tests := []struct {
+ name string // test subname
+ view dashboardView // one of htmlView{}, jsonView{}, or failuresView{}
+ req *apipb.DashboardRequest // what we pretend we sent to maintner
+ res *apipb.DashboardResponse // what we pretend we got back from maintner
+ testCommitData map[string]*Commit // what we pretend we loaded from datastore
+ want *uiTemplateData // what we're hoping we generated for the view/template
+ }{
+ // Basic test.
+ {
+ name: "html,zero_value_req,no_commits",
+ view: htmlView{},
+ req: &apipb.DashboardRequest{},
+ res: &apipb.DashboardResponse{
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ },
+ want: &uiTemplateData{
+ Dashboard: goDash,
+ Package: &Package{Name: "Go", Path: ""},
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ Builders: []string{"linux-386", "linux-amd64"},
+ Pagination: &Pagination{},
+ },
+ },
+
+ // Basic test + two commits: one that's in datastore and one that's not.
+ {
+ name: "html,zero_value_req,has_commit",
+ view: htmlView{},
+ req: &apipb.DashboardRequest{},
+ // Have only one commit load from the datastore:
+ testCommitData: map[string]*Commit{
+ "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac": &Commit{
+ PackagePath: "",
+ Hash: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ ResultData: []string{
+ "openbsd-amd64|true||", // pretend openbsd-amd64 passed (and thus exists)
+ },
+ },
+ },
+ res: &apipb.DashboardResponse{
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ Commits: []*apipb.DashCommit{
+ // This is the maintner commit response that is in the datastore:
+ {
+ Commit: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ AuthorName: "Foo Bar",
+ AuthorEmail: "foo@example.com",
+ CommitTimeSec: 1257894001,
+ Title: "runtime: fix all the bugs",
+ Branch: "master",
+ },
+ // And another commit that's not in the datastore:
+ {
+ Commit: "ffffffffffffffffffffffffffffffffffffffff",
+ AuthorName: "Fancy Fred",
+ AuthorEmail: "f@eff.tld",
+ CommitTimeSec: 1257894000,
+ Title: "all: add effs",
+ Branch: "master",
+ },
+ },
+ CommitsTruncated: true, // pretend there's a page 2
+ },
+ want: &uiTemplateData{
+ Dashboard: goDash,
+ Package: &Package{Name: "Go", Path: ""},
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ Builders: []string{"linux-386", "linux-amd64", "openbsd-amd64"},
+ Pagination: &Pagination{
+ Next: 1,
+ },
+ Commits: []*CommitInfo{
+ {
+ Hash: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ User: "Foo Bar <foo@example.com>",
+ Desc: "runtime: fix all the bugs",
+ Time: time.Unix(1257894001, 0),
+ ResultData: []string{
+ "openbsd-amd64|true||",
+ },
+ },
+ {
+ Hash: "ffffffffffffffffffffffffffffffffffffffff",
+ User: "Fancy Fred <f@eff.tld>",
+ Desc: "all: add effs",
+ Time: time.Unix(1257894000, 0),
+ },
+ },
+ },
+ },
+
+ // Test that we generate the TagState (sections at
+ // bottom with the x/foo repo state).
+ {
+ name: "html,zero_value_req,has_xrepos",
+ view: htmlView{},
+ req: &apipb.DashboardRequest{},
+ testCommitData: map[string]*Commit{},
+ res: &apipb.DashboardResponse{
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ Commits: []*apipb.DashCommit{
+ {
+ Commit: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ AuthorName: "Foo Bar",
+ AuthorEmail: "foo@example.com",
+ CommitTimeSec: 1257894001,
+ Title: "runtime: fix all the bugs",
+ Branch: "master",
+ },
+ },
+ Releases: []*apipb.GoRelease{
+ {
+ BranchName: "master",
+ BranchCommit: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ },
+ {
+ BranchName: "release-branch.go1.99",
+ BranchCommit: "ffffffffffffffffffffffffffffffffffffffff",
+ },
+ },
+ RepoHeads: []*apipb.DashRepoHead{
+ {
+ GerritProject: "go",
+ Commit: &apipb.DashCommit{
+ Commit: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ AuthorName: "Foo Bar",
+ AuthorEmail: "foo@example.com",
+ CommitTimeSec: 1257894001,
+ Title: "runtime: fix all the bugs",
+ Branch: "master",
+ },
+ },
+ {
+ GerritProject: "net",
+ Commit: &apipb.DashCommit{
+ Commit: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+ AuthorName: "Ee Yore",
+ AuthorEmail: "e@e.net",
+ CommitTimeSec: 1257894001,
+ Title: "all: fix networking",
+ Branch: "master",
+ },
+ },
+ {
+ GerritProject: "sys",
+ Commit: &apipb.DashCommit{
+ Commit: "dddddddddddddddddddddddddddddddddddddddd",
+ AuthorName: "Sys Tem",
+ AuthorEmail: "sys@s.net",
+ CommitTimeSec: 1257894001,
+ Title: "sys: support more systems",
+ Branch: "master",
+ },
+ },
+ },
+ },
+ want: &uiTemplateData{
+ Dashboard: goDash,
+ Package: &Package{Name: "Go", Path: ""},
+ Branches: []string{"release.foo", "release.bar", "dev.blah"},
+ Builders: []string{"linux-386", "linux-amd64"},
+ Pagination: &Pagination{},
+ Commits: []*CommitInfo{
+ {
+ Hash: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ User: "Foo Bar <foo@example.com>",
+ Desc: "runtime: fix all the bugs",
+ Time: time.Unix(1257894001, 0),
+ },
+ },
+ TagState: []*TagState{
+ {
+ Name: "master",
+ Tag: &CommitInfo{Hash: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac"},
+ Packages: []*PackageState{
+ {
+ Package: &Package{Name: "net", Path: "golang.org/x/net"},
+ Commit: &CommitInfo{
+ Hash: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+ PackagePath: "golang.org/x/net",
+ User: "Ee Yore <e@e.net>",
+ Desc: "all: fix networking",
+ Time: time.Unix(1257894001, 0),
+ },
+ },
+ {
+ Package: &Package{Name: "sys", Path: "golang.org/x/sys"},
+ Commit: &CommitInfo{
+ Hash: "dddddddddddddddddddddddddddddddddddddddd",
+ PackagePath: "golang.org/x/sys",
+ User: "Sys Tem <sys@s.net>",
+ Desc: "sys: support more systems",
+ Time: time.Unix(1257894001, 0),
+ },
+ },
+ },
+ },
+ {
+ Name: "release-branch.go1.99",
+ Tag: &CommitInfo{Hash: "ffffffffffffffffffffffffffffffffffffffff"},
+ Packages: []*PackageState{
+ {
+ Package: &Package{Name: "net", Path: "golang.org/x/net"},
+ Commit: &CommitInfo{
+ Hash: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+ PackagePath: "golang.org/x/net",
+ User: "Ee Yore <e@e.net>",
+ Desc: "all: fix networking",
+ Time: time.Unix(1257894001, 0),
+ },
+ },
+ {
+ Package: &Package{Name: "sys", Path: "golang.org/x/sys"},
+ Commit: &CommitInfo{
+ Hash: "dddddddddddddddddddddddddddddddddddddddd",
+ PackagePath: "golang.org/x/sys",
+ User: "Sys Tem <sys@s.net>",
+ Desc: "sys: support more systems",
+ Time: time.Unix(1257894001, 0),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+
+ // Test viewing a non-go repo.
+ {
+ name: "html,other_repo",
+ view: htmlView{},
+ req: &apipb.DashboardRequest{Repo: "golang.org/x/net"},
+ testCommitData: map[string]*Commit{},
+ res: &apipb.DashboardResponse{
+ Branches: []string{"master", "dev.blah"},
+ Commits: []*apipb.DashCommit{
+ {
+ Commit: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ AuthorName: "Foo Bar",
+ AuthorEmail: "foo@example.com",
+ CommitTimeSec: 1257894001,
+ Title: "net: fix all the bugs",
+ Branch: "master",
+ GoCommitAtTime: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ GoCommitLatest: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ },
+ },
+ },
+ want: &uiTemplateData{
+ Dashboard: goDash,
+ Package: &Package{Name: "net", Path: "golang.org/x/net"},
+ Branches: []string{"master", "dev.blah"},
+ Builders: []string{"linux-386", "linux-amd64"},
+ Pagination: &Pagination{},
+ Commits: []*CommitInfo{
+ {
+ PackagePath: "golang.org/x/net",
+ Hash: "26957168c4c0cdcc7ca4f0b19d0eb19474d224ac",
+ User: "Foo Bar <foo@example.com>",
+ Desc: "net: fix all the bugs",
+ Time: time.Unix(1257894001, 0),
+ ResultData: []string{
+ "|false||aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "|false||bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ },
+ },
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tb := newUITemplateDataBuilder(tt.view, tt.req, tt.res)
+ if tt.testCommitData != nil {
+ tb.testCommitData = tt.testCommitData
+ }
+ data, err := tb.buildTemplateData(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ diff := cmp.Diff(tt.want, data, cmpopts.IgnoreUnexported(CommitInfo{}))
+ if diff != "" {
+ t.Errorf("mismatch want->got:\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/app/cache/cache.go b/app/cache/cache.go
deleted file mode 100644
index 2aa2603..0000000
--- a/app/cache/cache.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cache
-
-import (
- "bytes"
- "compress/gzip"
- "context"
- "encoding/gob"
- "fmt"
- "net/http"
- "time"
-
- "google.golang.org/appengine/log"
- "google.golang.org/appengine/memcache"
-)
-
-// TimeKey specifies the memcache entity that keeps the logical datastore time.
-var TimeKey = "cachetime"
-
-const (
- nocache = "nocache"
- expiry = 10 * time.Minute
-)
-
-func newTime() uint64 { return uint64(time.Now().Unix()) << 32 }
-
-// Now returns the current logical datastore time to use for cache lookups.
-func Now(c context.Context) uint64 {
- t, err := memcache.Increment(c, TimeKey, 0, newTime())
- if err != nil {
- log.Errorf(c, "cache.Now: %v", err)
- return 0
- }
- return t
-}
-
-// Tick sets the current logical datastore time to a never-before-used time
-// and returns that time. It should be called to invalidate the cache.
-func Tick(c context.Context) uint64 {
- t, err := memcache.Increment(c, TimeKey, 1, newTime())
- if err != nil {
- log.Errorf(c, "cache.Tick: %v", err)
- return 0
- }
- return t
-}
-
-// Get fetches data for name at time now from memcache and unmarshals it into
-// value. It reports whether it found the cache record and logs any errors to
-// the admin console.
-func Get(c context.Context, r *http.Request, now uint64, name string, value interface{}) bool {
- if now == 0 || r.FormValue(nocache) != "" {
- return false
- }
- key := fmt.Sprintf("%s.%d", name, now)
- _, err := gzipGobCodec.Get(c, key, value)
- if err == nil {
- log.Debugf(c, "cache hit %q", key)
- return true
- }
- log.Debugf(c, "cache miss %q", key)
- if err != memcache.ErrCacheMiss {
- log.Errorf(c, "get cache %q: %v", key, err)
- }
- return false
-}
-
-// Set puts value into memcache under name at time now.
-// It logs any errors to the admin console.
-func Set(c context.Context, r *http.Request, now uint64, name string, value interface{}) {
- if now == 0 || r.FormValue(nocache) != "" {
- return
- }
- key := fmt.Sprintf("%s.%d", name, now)
- err := gzipGobCodec.Set(c, &memcache.Item{
- Key: key,
- Object: value,
- Expiration: expiry,
- })
- if err != nil {
- log.Errorf(c, "set cache %q: %v", key, err)
- }
-}
-
-var gzipGobCodec = memcache.Codec{Marshal: marshal, Unmarshal: unmarshal}
-
-func marshal(v interface{}) ([]byte, error) {
- var b bytes.Buffer
- zw := gzip.NewWriter(&b)
- if err := gob.NewEncoder(zw).Encode(v); err != nil {
- return nil, err
- }
- if err := zw.Close(); err != nil {
- return nil, err
- }
- return b.Bytes(), nil
-}
-
-func unmarshal(b []byte, v interface{}) error {
- zr, err := gzip.NewReader(bytes.NewReader(b))
- if err != nil {
- return err
- }
- return gob.NewDecoder(zr).Decode(v)
-}
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
index cad470a..c1aa34f 100644
--- a/cmd/coordinator/coordinator.go
+++ b/cmd/coordinator/coordinator.go
@@ -62,6 +62,7 @@
"golang.org/x/build/internal/sourcecache"
"golang.org/x/build/livelog"
"golang.org/x/build/maintner/maintnerd/apipb"
+ "golang.org/x/build/repos"
revdialv2 "golang.org/x/build/revdial/v2"
"golang.org/x/build/types"
"golang.org/x/crypto/acme/autocert"
@@ -488,6 +489,13 @@
if isBuilding(rev) {
return false
}
+ if rev.SubName != "" {
+ // Don't build repos we don't know about,
+ // so importPathOfRepo won't panic later.
+ if r, ok := repos.ByGerritProject[rev.SubName]; !ok || r.ImportPath == "" || !r.CoordinatorCanBuild {
+ return false
+ }
+ }
buildConf, ok := dashboard.Builders[rev.Name]
if !ok {
if logUnknownBuilder.Allow() {
@@ -904,7 +912,10 @@
// post-submit work to do. It's called in a loop by findWorkLoop.
func findWork() error {
var bs types.BuildStatus
- if err := dash("GET", "", url.Values{"mode": {"json"}}, nil, &bs); err != nil {
+ if err := dash("GET", "", url.Values{
+ "mode": {"json"},
+ "branch": {"mixed"},
+ }, nil, &bs); err != nil {
return err
}
knownToDashboard := map[string]bool{} // keys are builder
@@ -933,6 +944,9 @@
}
for _, br := range bs.Revisions {
+ if r, ok := repos.ByGerritProject[br.Repo]; !ok || !r.CoordinatorCanBuild {
+ continue
+ }
if br.Repo == "grpc-review" {
// Skip the grpc repo. It's only for reviews
// for now (using LetsUseGerrit).
@@ -1055,8 +1069,7 @@
log.Printf("Warning: skipping incomplete %#v", work)
continue
}
- if work.Project == "grpc-review" {
- // Skip grpc-review, which is only for reviews for now.
+ if r, ok := repos.ByGerritProject[work.Project]; !ok || !r.CoordinatorCanBuild {
continue
}
key := tryWorkItemKey(work)
@@ -3898,18 +3911,25 @@
}
// importPathOfRepo returns the Go import path corresponding to the
-// root of the given repo (Gerrit project). Because it's a Go import
-// path, it always has forward slashes and no trailing slash.
+// root of the given non-"go" repo (Gerrit project). Because it's a Go
+// import path, it always has forward slashes and no trailing slash.
//
// For example:
// "net" -> "golang.org/x/net"
// "crypto" -> "golang.org/x/crypto"
// "dl" -> "golang.org/dl"
func importPathOfRepo(repo string) string {
- if repo == "dl" {
- return "golang.org/dl"
+ r := repos.ByGerritProject[repo]
+ if r == nil {
+ // mayBuildRev prevents adding work for repos we don't know about,
+ // so this shouldn't happen. If it does, a panic will be useful.
+ panic(fmt.Sprintf("importPathOfRepo(%q) on unknown repo %q", repo, repo))
}
- return "golang.org/x/" + repo
+ if r.ImportPath == "" {
+ // Likewise. This shouldn't happen.
+ panic(fmt.Sprintf("importPathOfRepo(%q) doesn't have an ImportPath", repo))
+ }
+ return r.ImportPath
}
// slowBotsFromComments looks at the TRY= comments from Gerrit (in
diff --git a/cmd/gitmirror/gitmirror.go b/cmd/gitmirror/gitmirror.go
index 44bb9c3..7a5ea4c 100644
--- a/cmd/gitmirror/gitmirror.go
+++ b/cmd/gitmirror/gitmirror.go
@@ -3,18 +3,15 @@
// license that can be found in the LICENSE file.
// The gitmirror binary watches the specified Gerrit repositories for
-// new commits and reports them to the build dashboard.
+// new commits and syncs them to GitHub.
//
-// It also serves tarballs over HTTP for the build system, and pushes
-// new commits to GitHub.
+// It also serves tarballs over HTTP for the build system.
package main
import (
"bytes"
"context"
"crypto/sha1"
- "encoding/json"
- "errors"
"flag"
"fmt"
"io"
@@ -22,7 +19,6 @@
"log"
"net"
"net/http"
- "net/url"
"os"
"os/exec"
"path"
@@ -35,78 +31,32 @@
"time"
"cloud.google.com/go/compute/metadata"
- "golang.org/x/build/buildenv"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/gitauth"
"golang.org/x/build/maintner"
"golang.org/x/build/maintner/godata"
+ repospkg "golang.org/x/build/repos"
)
const (
- goBase = "https://go.googlesource.com/"
- watcherVersion = 3 // must match dashboard/app/build/handler.go's watcherVersion
- master = "master" // name of the master branch
+ goBase = "https://go.googlesource.com/"
)
var (
- httpAddr = flag.String("http", "", "If non-empty, the listen address to run an HTTP server on")
- cacheDir = flag.String("cachedir", "", "git cache directory. If empty a temp directory is made.")
-
- dashFlag = flag.String("dash", "", "Dashboard URL (must end in /). If unset, will be automatically derived from the GCE project name.")
- keyFile = flag.String("key", defaultKeyFile, "Build dashboard key file. If empty, automatic from GCE project metadata")
-
+ httpAddr = flag.String("http", "", "If non-empty, the listen address to run an HTTP server on")
+ cacheDir = flag.String("cachedir", "", "git cache directory. If empty a temp directory is made.")
pollInterval = flag.Duration("poll", 60*time.Second, "Remote repo poll interval")
-
- // TODO(bradfitz): these three are all kinda the same and
- // redundant. Unify after research.
- network = flag.Bool("network", true, "Enable network calls (disable for testing)")
- mirror = flag.Bool("mirror", false, "whether to mirror to github")
- report = flag.Bool("report", true, "Report updates to build dashboard (use false for development dry-run mode)")
-
- filter = flag.String("filter", "", "If non-empty, a comma-separated list of directories or files to watch for new commits (only works on main repo). If empty, watch all files in repo.")
- branches = flag.String("branches", "", "If non-empty, a comma-separated list of branches to watch. If empty, watch changes on every branch.")
+ mirror = flag.Bool("mirror", false, "whether to mirror to github; if disabled, it only runs in HTTP archive server mode")
)
-var (
- defaultKeyFile = filepath.Join(homeDir(), ".gobuildkey")
- dashboardKey = ""
- networkSeen = make(map[string]bool) // testing mode only (-network=false); known hashes
-)
-
-var httpClient = &http.Client{
- Timeout: 30 * time.Second, // overkill
-}
-
var gerritClient = gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
-var (
- // gitLogFn returns the list of unseen Commits on a Repo,
- // typically by shelling out to `git log`.
- // gitLogFn is a global var so we can stub it out for tests.
- gitLogFn = gitLog
-
- // gitRemotesFn returns a slice of remote branches known to the git repo.
- // gitRemotesFn is a global var so we can stub it out for tests.
- gitRemotesFn = gitRemotes
-)
-
func main() {
flag.Parse()
if err := gitauth.Init(); err != nil {
log.Fatalf("gitauth: %v", err)
}
- if *dashFlag == "" && metadata.OnGCE() {
- project, err := metadata.ProjectID()
- if err != nil {
- log.Fatalf("metadata.ProjectID: %v", err)
- }
- *dashFlag = buildenv.ByProjectID(project).DashBase()
- }
- if *dashFlag == "" {
- log.Fatal("-dash must be specified and could not be autodetected")
- }
-
log.Printf("gitmirror running.")
go pollGerritAndTickle()
@@ -118,10 +68,6 @@
// runGitMirror is a little wrapper so we can use defer and return to signal
// errors. It should only return a non-nil error.
func runGitMirror() error {
- if !strings.HasSuffix(*dashFlag, "/") {
- return errors.New("dashboard URL (-dashboard) must end in /")
- }
-
if *mirror {
sshDir := filepath.Join(homeDir(), ".ssh")
sshKey := filepath.Join(sshDir, "id_ed25519")
@@ -165,14 +111,6 @@
}
}
- if *report {
- if k, err := readKey(); err != nil {
- return err
- } else {
- dashboardKey = k
- }
- }
-
if *httpAddr != "" {
http.HandleFunc("/debug/env", handleDebugEnv)
http.HandleFunc("/debug/goroutines", handleDebugGoroutines)
@@ -185,12 +123,7 @@
errc := make(chan error)
- subrepos, err := subrepoList()
- if err != nil {
- return err
- }
-
- startRepo := func(name, path string, dash bool) {
+ startRepo := func(name string) {
log.Printf("Starting watch of repo %s", name)
url := goBase + name
var dst string
@@ -202,7 +135,7 @@
log.Printf("Not mirroring repo %s", name)
}
}
- r, err := NewRepo(url, dst, path, dash)
+ r, err := NewRepo(url, dst)
if err != nil {
errc <- err
return
@@ -215,34 +148,13 @@
r.Loop()
}
- go startRepo("go", "", true)
-
- seen := map[string]bool{"go": true}
- for _, path := range subrepos {
- name := strings.TrimPrefix(path, "golang.org/x/")
- seen[name] = true
- go startRepo(name, path, true)
- }
if *mirror {
gerritRepos, err := gerritMetaMap()
if err != nil {
return fmt.Errorf("gerritMetaMap: %v", err)
}
for name := range gerritRepos {
- if seen[name] {
- // Repo already picked up by dashboard list.
- continue
- }
- path := "golang.org/x/" + name
- switch name {
- case "dl":
- // This subrepo is different from others in that
- // it doesn't use the /x/ path element.
- path = "golang.org/" + name
- case "protobuf":
- path = "google.golang.org/" + name
- }
- go startRepo(name, path, false)
+ go startRepo(name)
}
}
@@ -278,58 +190,13 @@
// shouldMirrorTo returns the GitHub repository the named repo should be
// mirrored to or "" if it should not be mirrored.
func shouldMirrorTo(name string) (dst string) {
- switch name {
- case
- "arch",
- "benchmarks",
- "blog",
- "build",
- "crypto",
- "debug",
- "dl",
- "example",
- "exp",
- "gddo",
- "go",
- "gofrontend",
- "image",
- "lint",
- "mobile",
- "mod",
- "net",
- "oauth2",
- "playground",
- "proposal",
- "review",
- "scratch",
- "sync",
- "sys",
- "talks",
- "term",
- "text",
- "time",
- "tools",
- "tour",
- "vgo",
- "website",
- "xerrors":
- // Mirror this.
- case "protobuf":
+ if name == "protobuf" {
return "git@github.com:protocolbuffers/protobuf-go.git"
- default:
- // Else, see if it appears to be a subrepo:
- r, err := httpClient.Get("https://golang.org/x/" + name)
- if err != nil {
- log.Printf("repo %v doesn't seem to exist: %v", name, err)
- return ""
- }
- r.Body.Close()
- if r.StatusCode/100 != 2 {
- return ""
- }
- // Mirror this.
}
- return "git@github.com:golang/" + name + ".git"
+ if r, ok := repospkg.ByGerritProject[name]; ok && r.MirroredToGithub {
+ return "git@github.com:golang/" + name + ".git"
+ }
+ return ""
}
// a statusEntry is a status string at a specific time.
@@ -375,13 +242,9 @@
// Repo represents a repository to be watched.
type Repo struct {
- root string // on-disk location of the git repo, *cacheDir/name
- path string // base import path for repo (blank for main repo)
- commits map[string]*Commit // keyed by full commit hash (40 lowercase hex digits)
- branches map[string]*Branch // keyed by branch name, eg "release-branch.go1.3" (or empty for default)
- dash bool // push new commits to the dashboard
- mirror bool // push new commits to 'dest' remote
- status statusRing
+ root string // on-disk location of the git repo, *cacheDir/name
+ mirror bool // push new commits to 'dest' remote
+ status statusRing
mu sync.Mutex
err error
@@ -396,20 +259,12 @@
//
// If dstURL is not empty, changes from the source repository will
// be mirrored to the specified destination repository.
-// The importPath argument is the base import path of the repository,
-// and should be empty for the main Go repo.
-// The dash argument should be set true if commits to this
-// repo should be reported to the build dashboard.
-func NewRepo(srcURL, dstURL, importPath string, dash bool) (*Repo, error) {
+func NewRepo(srcURL, dstURL string) (*Repo, error) {
name := path.Base(srcURL) // "go", "net", etc
root := filepath.Join(*cacheDir, name)
r := &Repo{
- path: importPath,
- root: root,
- commits: make(map[string]*Commit),
- branches: make(map[string]*Branch),
- mirror: dstURL != "",
- dash: dash,
+ root: root,
+ mirror: dstURL != "",
}
http.Handle("/debug/watcher/"+r.name(), r)
@@ -461,14 +316,6 @@
r.setStatus("added dest remote")
}
- if r.dash {
- r.logf("loading commit log")
- if err := r.update(false); err != nil {
- return nil, err
- }
- r.logf("found %v branches among %v commits\n", len(r.branches), len(r.commits))
- }
-
return r, nil
}
@@ -591,9 +438,8 @@
return f.Close()
}
-// Loop continuously runs "git fetch" in the repo, checks for
-// new commits, posts any new commits to the dashboard (if enabled),
-// and mirrors commits to a destination repo (if enabled).
+// Loop continuously runs "git fetch" in the repo, checks for new
+// commits and mirrors commits to a destination repo (if enabled).
func (r *Repo) Loop() {
tickler := repoTickler(r.name())
for {
@@ -611,14 +457,6 @@
continue
}
}
- if r.dash {
- if err := r.updateDashboard(); err != nil {
- r.logf("updateDashboard failed in repo loop: %v", err)
- r.setErr(err)
- time.Sleep(10 * time.Second)
- continue
- }
- }
r.setErr(nil)
r.setStatus("waiting")
@@ -636,459 +474,14 @@
}
}
-func (r *Repo) updateDashboard() (err error) {
- r.setStatus("updating dashboard")
- defer func() {
- if err == nil {
- r.setStatus("updated dashboard")
- }
- }()
- if err := r.update(true); err != nil {
- return err
- }
- remotes, err := gitRemotesFn(r)
- if err != nil {
- return err
- }
- for _, name := range remotes {
- b, ok := r.branches[name]
- if !ok {
- // skip branch; must be already merged
- continue
- }
- if err := r.postNewCommits(b); err != nil {
- return err
- }
- }
- return nil
-}
-
func (r *Repo) name() string {
- if r.path == "" {
- return "go"
- }
- return path.Base(r.path)
+ return filepath.Base(r.root)
}
func (r *Repo) logf(format string, args ...interface{}) {
log.Printf(r.name()+": "+format, args...)
}
-// postNewCommits looks for unseen commits on the specified branch and
-// posts them to the dashboard.
-func (r *Repo) postNewCommits(b *Branch) error {
- if b.Head == b.LastSeen {
- return nil
- }
- c := b.LastSeen
- if c == nil {
- // Haven't seen anything on this branch yet:
- if b.Name == master {
- // For the master branch, bootstrap by creating a dummy
- // commit with a lone child that is the initial commit.
- c = &Commit{}
- for _, c2 := range r.commits {
- if c2.Parent == "" {
- c.children = []*Commit{c2}
- break
- }
- }
- if c.children == nil {
- return fmt.Errorf("couldn't find initial commit")
- }
- } else {
- // Find the commit that this branch forked from.
- base, err := r.mergeBase("heads/"+b.Name, master)
- if err != nil {
- return err
- }
- var ok bool
- c, ok = r.commits[base]
- if !ok {
- return fmt.Errorf("couldn't find base commit: %v", base)
- }
- }
- }
- if err := r.postChildren(b, c); err != nil {
- return err
- }
- b.LastSeen = b.Head
- return nil
-}
-
-// postChildren posts to the dashboard all descendants of the given parent.
-// It ignores descendants that are not on the given branch.
-func (r *Repo) postChildren(b *Branch, parent *Commit) error {
- for _, c := range parent.children {
- if c.Branch != b.Name {
- continue
- }
- if err := r.postCommit(c); err != nil {
- if strings.Contains(err.Error(), "this package already has a first commit; aborting") {
- return nil
- }
- return err
- }
- }
- for _, c := range parent.children {
- if err := r.postChildren(b, c); err != nil {
- return err
- }
- }
- return nil
-}
-
-// postCommit sends a commit to the build dashboard.
-func (r *Repo) postCommit(c *Commit) error {
- if !*report {
- r.logf("dry-run mode; NOT posting commit to dashboard: %v", c)
- return nil
- }
- r.logf("sending commit to dashboard: %v", c)
-
- t, err := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", c.Date)
- if err != nil {
- return fmt.Errorf("postCommit: parsing date %q for commit %v: %v", c.Date, c, err)
- }
- dc := struct {
- PackagePath string // (empty for main repo commits)
- Hash string
- ParentHash string
-
- User string
- Desc string
- Time time.Time
- Branch string
-
- NeedsBenchmarking bool
- }{
- PackagePath: r.path,
- Hash: c.Hash,
- ParentHash: c.Parent,
-
- User: c.Author,
- Desc: c.Desc,
- Time: t,
- Branch: c.Branch,
-
- NeedsBenchmarking: c.NeedsBenchmarking(),
- }
- b, err := json.Marshal(dc)
- if err != nil {
- return fmt.Errorf("postCommit: marshaling request body: %v", err)
- }
-
- if !*network {
- if c.Parent != "" {
- if !networkSeen[c.Parent] {
- r.logf("%v: %v", c.Parent, r.commits[c.Parent])
- return fmt.Errorf("postCommit: no parent %v found on dashboard for %v", c.Parent, c)
- }
- }
- if networkSeen[c.Hash] {
- return fmt.Errorf("postCommit: already seen %v", c)
- }
- networkSeen[c.Hash] = true
- return nil
- }
-
- v := url.Values{"version": {fmt.Sprint(watcherVersion)}, "key": {dashboardKey}}
- u := *dashFlag + "commit?" + v.Encode()
- resp, err := http.Post(u, "text/json", bytes.NewReader(b))
- if err != nil {
- return err
- }
- body, err := ioutil.ReadAll(resp.Body)
- resp.Body.Close()
- if err != nil {
- return fmt.Errorf("postCommit: reading body: %v", err)
- }
- if resp.StatusCode != 200 {
- return fmt.Errorf("postCommit: status: %v\nbody: %s", resp.Status, body)
- }
-
- var s struct {
- Error string
- }
- if err := json.Unmarshal(body, &s); err != nil {
- return fmt.Errorf("postCommit: decoding response: %v", err)
- }
- if s.Error != "" {
- return fmt.Errorf("postCommit: error: %v", s.Error)
- }
- return nil
-}
-
-// update looks for new commits and branches,
-// and updates the commits and branches maps.
-func (r *Repo) update(noisy bool) error {
- remotes, err := gitRemotesFn(r)
- if err != nil {
- return err
- }
- for _, name := range remotes {
- b := r.branches[name]
-
- // Find all unseen commits on this branch.
- revspec := "heads/" + name
- if b != nil {
- // If we know about this branch,
- // only log commits down to the known head.
- revspec = b.Head.Hash + ".." + revspec
- }
- log, err := gitLogFn(r, "--topo-order", revspec)
- if err != nil {
- return err
- }
- if len(log) == 0 {
- // No commits to handle; carry on.
- continue
- }
-
- var nDups, nDrops int
-
- // Add unknown commits to r.commits.
- var added []*Commit
- for _, c := range log {
- if noisy {
- r.logf("found new commit %v", c)
- }
- // If we've already seen this commit,
- // only store the master one in r.commits.
- if _, ok := r.commits[c.Hash]; ok {
- nDups++
- if name != master {
- nDrops++
- continue
- }
- }
- c.Branch = name
- r.commits[c.Hash] = c
- added = append(added, c)
- }
-
- if nDups > 0 {
- r.logf("saw %v duplicate commits; dropped %v of them", nDups, nDrops)
- }
-
- // Link added commits.
- for _, c := range added {
- if c.Parent == "" {
- // This is the initial commit; no parent.
- r.logf("no parents for initial commit %v", c)
- continue
- }
- // Find parent commit.
- p, ok := r.commits[c.Parent]
- if !ok {
- return fmt.Errorf("can't find parent %q for %v", c.Parent, c)
- }
- // Link parent Commit.
- c.parent = p
- // Link child Commits.
- p.children = append(p.children, c)
- }
-
- // Update branch head, or add newly discovered branch.
- // If we had already seen log[0], eg. on a different branch,
- // we would never have linked it in the loop above.
- // We therefore fetch the commit from r.commits to ensure we have
- // the right address.
- head := r.commits[log[0].Hash]
- if b != nil {
- // Known branch; update head.
- b.Head = head
- r.logf("updated branch head: %v", b)
- } else {
- // It's a new branch; add it.
- seen, err := r.lastSeen(head.Hash)
- if err != nil {
- return err
- }
- b = &Branch{Name: name, Head: head, LastSeen: seen}
- r.branches[name] = b
- r.logf("found branch: %v", b)
- }
- }
-
- return nil
-}
-
-// lastSeen finds the most recent commit the dashboard has seen,
-// starting at the specified head. If the dashboard hasn't seen
-// any of the commits from head to the beginning, it returns nil.
-func (r *Repo) lastSeen(head string) (*Commit, error) {
- h, ok := r.commits[head]
- if !ok {
- return nil, fmt.Errorf("lastSeen: can't find %q in commits", head)
- }
-
- var s []*Commit
- for c := h; c != nil; c = c.parent {
- s = append(s, c)
- }
-
- var err error
- i := sort.Search(len(s), func(i int) bool {
- if err != nil {
- return false
- }
- ok, err = r.dashSeen(s[i].Hash)
- return ok
- })
- switch {
- case err != nil:
- return nil, fmt.Errorf("lastSeen: %v", err)
- case i < len(s):
- return s[i], nil
- default:
- // Dashboard saw no commits.
- return nil, nil
- }
-}
-
-// dashSeen reports whether the build dashboard knows the specified commit.
-func (r *Repo) dashSeen(hash string) (bool, error) {
- if !*network {
- return networkSeen[hash], nil
- }
- v := url.Values{"hash": {hash}, "packagePath": {r.path}}
- u := *dashFlag + "commit?" + v.Encode()
- resp, err := httpClient.Get(u)
- if err != nil {
- return false, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- return false, fmt.Errorf("status: %v", resp.Status)
- }
- var s struct {
- Error string
- }
- err = json.NewDecoder(resp.Body).Decode(&s)
- if err != nil {
- return false, err
- }
- switch s.Error {
- case "":
- // Found one.
- return true, nil
- case "Commit not found":
- // Commit not found, keep looking for earlier commits.
- return false, nil
- default:
- return false, fmt.Errorf("dashboard: %v", s.Error)
- }
-}
-
-// mergeBase returns the hash of the merge base for revspecs a and b.
-func (r *Repo) mergeBase(a, b string) (string, error) {
- cmd := exec.Command("git", "merge-base", a, b)
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return "", fmt.Errorf("git merge-base %s..%s: %v", a, b, err)
- }
- return string(bytes.TrimSpace(out)), nil
-}
-
-// gitRemotes returns a slice of remote branches known to the git repo.
-// It always puts "origin/master" first.
-func gitRemotes(r *Repo) ([]string, error) {
- if *branches != "" {
- return strings.Split(*branches, ","), nil
- }
-
- cmd := exec.Command("git", "branch")
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return nil, fmt.Errorf("git branch: %v", err)
- }
- bs := []string{master}
- for _, b := range strings.Split(string(out), "\n") {
- b = strings.TrimPrefix(b, "* ")
- b = strings.TrimSpace(b)
- // Ignore aliases, blank lines, and master (it's already in bs).
- if b == "" || strings.Contains(b, "->") || b == master {
- continue
- }
- // Ignore pre-go1 release branches; they are just noise.
- if strings.HasPrefix(b, "release-branch.r") {
- continue
- }
- bs = append(bs, b)
- }
- return bs, nil
-}
-
-const logFormat = `--format=format:` + logBoundary + `%H
-%P
-%an <%ae>
-%cD
-%B
-` + fileBoundary
-
-const logBoundary = `_-_- magic boundary -_-_`
-const fileBoundary = `_-_- file boundary -_-_`
-
-// gitLog runs "git log" with the supplied arguments
-// and parses the output into Commit values.
-func gitLog(r *Repo, dir string, args ...string) ([]*Commit, error) {
- args = append([]string{"log", "--date=rfc", "--name-only", "--parents", logFormat}, args...)
- if r.path == "" && *filter != "" {
- paths := strings.Split(*filter, ",")
- args = append(args, "--")
- args = append(args, paths...)
- }
- cmd := exec.Command("git", args...)
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return nil, fmt.Errorf("git %v: %v\n%s", strings.Join(args, " "), err, out)
- }
-
- // We have a commit with description that contains 0x1b byte.
- // Mercurial does not escape it, but xml.Unmarshal does not accept it.
- // TODO(adg): do we still need to scrub this? Probably.
- out = bytes.Replace(out, []byte{0x1b}, []byte{'?'}, -1)
-
- var cs []*Commit
- for _, text := range strings.Split(string(out), logBoundary) {
- text = strings.TrimSpace(text)
- if text == "" {
- continue
- }
- p := strings.SplitN(text, "\n", 5)
- if len(p) != 5 {
- return nil, fmt.Errorf("git log %v: malformed commit: %q", strings.Join(args, " "), text)
- }
-
- // The change summary contains the change description and files
- // modified in this commit. There is no way to directly refer
- // to the modified files in the log formatting string, so we look
- // for the file boundary after the description.
- changeSummary := p[4]
- descAndFiles := strings.SplitN(changeSummary, fileBoundary, 2)
- desc := strings.TrimSpace(descAndFiles[0])
-
- // For branch merges, the list of files can still be empty
- // because there are no changed files.
- files := strings.Replace(strings.TrimSpace(descAndFiles[1]), "\n", " ", -1)
-
- cs = append(cs, &Commit{
- Hash: p[0],
- // TODO(adg): This may break with branch merges.
- Parent: strings.Split(p[1], " ")[0],
- Author: p[2],
- Date: p[3],
- Desc: desc,
- Files: files,
- })
- }
- return cs, nil
-}
-
// fetch runs "git fetch" in the repository root.
// It tries three times, just in case it failed because of a transient error.
func (r *Repo) fetch() (err error) {
@@ -1251,58 +644,6 @@
return err
}
-// Branch represents a Mercurial branch.
-type Branch struct {
- Name string
- Head *Commit
- LastSeen *Commit // the last commit posted to the dashboard
-}
-
-func (b *Branch) String() string {
- return fmt.Sprintf("%q(Head: %v LastSeen: %v)", b.Name, b.Head, b.LastSeen)
-}
-
-// Commit represents a single Git commit.
-type Commit struct {
- Hash string
- Author string
- Date string // Format: "Mon, 2 Jan 2006 15:04:05 -0700"
- Desc string // Plain text, first line is a short description.
- Parent string
- Branch string
- Files string
-
- // For walking the graph.
- parent *Commit
- children []*Commit
-}
-
-func (c *Commit) String() string {
- s := c.Hash
- if c.Branch != "" {
- s += fmt.Sprintf("[%v]", c.Branch)
- }
- s += fmt.Sprintf("(%q)", strings.SplitN(c.Desc, "\n", 2)[0])
- return s
-}
-
-// NeedsBenchmarking reports whether the Commit needs benchmarking.
-func (c *Commit) NeedsBenchmarking() bool {
- // Do not benchmark branch commits, they are usually not interesting
- // and fall out of the trunk succession.
- if c.Branch != master {
- return false
- }
- // Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
- for _, f := range strings.Split(c.Files, " ") {
- if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
- !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
- return true
- }
- }
- return false
-}
-
func homeDir() string {
switch runtime.GOOS {
case "plan9":
@@ -1313,57 +654,6 @@
return os.Getenv("HOME")
}
-func readKey() (string, error) {
- c, err := ioutil.ReadFile(*keyFile)
- if os.IsNotExist(err) && metadata.OnGCE() {
- key, err := metadata.ProjectAttributeValue("builder-master-key")
- if err != nil {
- return "", fmt.Errorf("-key=%s doesn't exist, and key can't be loaded from GCE metadata: %v", *keyFile, err)
- }
- return strings.TrimSpace(key), nil
- }
- if err != nil {
- return "", err
- }
- return string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0])), nil
-}
-
-// subrepoList fetches a list of sub-repositories from the dashboard
-// and returns them as a slice of base import paths.
-// Eg, []string{"golang.org/x/tools", "golang.org/x/net"}.
-func subrepoList() ([]string, error) {
- if !*network {
- return nil, nil
- }
-
- r, err := httpClient.Get(*dashFlag + "packages?kind=subrepo")
- if err != nil {
- return nil, fmt.Errorf("subrepo list: %v", err)
- }
- defer r.Body.Close()
- if r.StatusCode != 200 {
- return nil, fmt.Errorf("subrepo list: got status %v", r.Status)
- }
- var resp struct {
- Response []struct {
- Path string
- }
- Error string
- }
- err = json.NewDecoder(r.Body).Decode(&resp)
- if err != nil {
- return nil, fmt.Errorf("subrepo list: %v", err)
- }
- if resp.Error != "" {
- return nil, fmt.Errorf("subrepo list: %v", resp.Error)
- }
- var pkgs []string
- for _, r := range resp.Response {
- pkgs = append(pkgs, r.Path)
- }
- return pkgs, nil
-}
-
var (
ticklerMu sync.Mutex
ticklers = make(map[string]chan bool)
diff --git a/cmd/gitmirror/gitmirror_test.go b/cmd/gitmirror/gitmirror_test.go
index 244e989..4831737 100644
--- a/cmd/gitmirror/gitmirror_test.go
+++ b/cmd/gitmirror/gitmirror_test.go
@@ -6,14 +6,44 @@
import (
"context"
- "fmt"
+ "io/ioutil"
+ "log"
"net/http/httptest"
+ "os"
"os/exec"
+ "path/filepath"
"reflect"
"strings"
"testing"
)
+func TestMain(m *testing.M) {
+ // The tests need a dummy directory that exists with a
+ // basename of "build", but the tests never write to it. So we
+ // can create one and share it for all tests.
+ tempDir, err := ioutil.TempDir("", "")
+ if err != nil {
+ log.Fatal(err)
+ }
+ tempRepoRoot = filepath.Join(tempDir, "build")
+ if err := os.Mkdir(tempRepoRoot, 0700); err != nil {
+ log.Fatal(err)
+ }
+
+ e := m.Run()
+ os.RemoveAll(tempDir)
+ os.Exit(e)
+}
+
+var tempRepoRoot string
+
+func newTestRepo() *Repo {
+ return &Repo{
+ root: tempRepoRoot,
+ mirror: false,
+ }
+}
+
func TestHomepage(t *testing.T) {
req := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder()
@@ -27,13 +57,13 @@
}
func TestDebugWatcher(t *testing.T) {
- r := &Repo{path: "build"}
+ r := newTestRepo()
r.setStatus("waiting")
req := httptest.NewRequest("GET", "/debug/watcher/build", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != 200 {
- t.Fatalf("GET /: want code 200, got %d", w.Code)
+ t.Fatalf("GET / = code %d, want 200", w.Code)
}
body := w.Body.String()
if substr := `watcher status for repo: "build"`; !strings.Contains(body, substr) {
@@ -70,7 +100,7 @@
f := &fakeCmd{}
testHookArchiveCmd = f.CommandContext
defer func() { testHookArchiveCmd = nil }()
- r := &Repo{path: "build"}
+ r := newTestRepo()
r.setStatus("waiting")
req := httptest.NewRequest("GET", "/build.tar.gz?rev=example-branch", nil)
w := httptest.NewRecorder()
@@ -97,7 +127,7 @@
testHookArchiveCmd = nil
testHookFetchCmd = nil
}()
- r := &Repo{path: "build"}
+ r := newTestRepo()
r.setStatus("waiting")
req := httptest.NewRequest("GET", "/build.tar.gz?rev=example-branch", nil)
w := httptest.NewRecorder()
@@ -113,110 +143,3 @@
t.Fatalf("cmd: want '%q' for args, got %q", wantArgs, f2.Args)
}
}
-
-// TestUpdate tests that we link new commits correctly in
-// our linked list (parent <=> child) of commits, and update
-// the Repo's map of commits correctly.
-func TestUpdate(t *testing.T) {
- oldNetwork := *network
- defer func() {
- *network = oldNetwork
- }()
-
- *network = false
-
- hash := func(i int) string {
- return fmt.Sprintf("abc123%d", i)
- }
-
- commit := func(i int) *Commit {
- c := &Commit{
- Hash: hash(i),
- Author: "Sarah Adams <shadams@google.com>",
- Date: "Fri, 15 Sep 2017 13:56:53 -0700",
- Desc: fmt.Sprintf("CONTRIBUTORS: add person %d.", i),
- Files: "CONTRIBUTORS",
- }
-
- if i > 0 {
- c.Parent = hash(i - 1)
- }
-
- return c
- }
-
- gitLogFn = func(r *Repo, dir string, args ...string) ([]*Commit, error) {
- // We are testing new commits on a non-master branch.
- // So, for simplicity, return no new commits on master.
- for _, a := range args {
- if strings.Contains(a, "origin/master") {
- return nil, nil
- }
- }
-
- var cs []*Commit
- for i := 0; i < 5; i++ {
- cs = append(cs, commit(i))
- }
- return cs, nil
- }
-
- gitRemotesFn = func(r *Repo) ([]string, error) {
- return []string{"origin/master", "origin/other.branch"}, nil
- }
-
- repo := newTestRepo()
-
- // Add a known commit on master.
- // This commit is HEAD of origin/master when we forked to
- // create the 'origin/other.branch' branch.
- head := commit(0)
- head.Branch = "origin/master"
- repo.commits[hash(0)] = head
-
- master := &Branch{
- Name: "origin/master",
- Head: head,
- LastSeen: head,
- }
- repo.branches["origin/master"] = master
-
- err := repo.update(false)
- if err != nil {
- t.Fatalf("update: got error %v", err)
- }
-
- head = repo.branches["origin/other.branch"].Head
-
- if head.Hash != hash(0) {
- t.Fatalf("expected head to have hash %s, got %s.", hash(0), head.Hash)
- }
-
- if len(head.children) != 1 {
- t.Fatalf("expected head to have 1 child commit, got %d.", len(head.children))
- }
-
- for i := 0; i < 5; i++ {
- if i != 0 {
- if repo.commits[hash(i)].parent == nil {
- t.Errorf("expected commit %d to have a parent commit.", i)
- }
- }
- if i != 4 {
- if len(repo.commits[hash(i)].children) == 0 {
- t.Errorf("expected commit %d to have child commits.", i)
- }
- }
- }
-}
-
-func newTestRepo() *Repo {
- return &Repo{
- path: "",
- root: "/usr/local/home/go",
- commits: make(map[string]*Commit),
- branches: make(map[string]*Branch),
- mirror: false,
- dash: false,
- }
-}
diff --git a/cmd/retrybuilds/retrybuilds.go b/cmd/retrybuilds/retrybuilds.go
index 1218d4d..51a22da 100644
--- a/cmd/retrybuilds/retrybuilds.go
+++ b/cmd/retrybuilds/retrybuilds.go
@@ -84,7 +84,16 @@
return
}
if *builder == "" {
- log.Fatalf("Missing -builder, -redo-flaky, or -loghash flag.")
+ log.Fatalf("Missing -builder, -redo-flaky, -substr, or -loghash flag.")
+ }
+ if *hash == "" {
+ for _, f := range failures() {
+ if f.Builder != *builder {
+ continue
+ }
+ wipe(f.Builder, f.Hash)
+ }
+ return
}
wipe(*builder, fullHash(*hash))
}
@@ -175,25 +184,23 @@
}
func fullHash(h string) string {
- if h == "" || len(h) == 40 {
+ if len(h) == 40 {
return h
}
- for _, f := range failures() {
- if strings.HasPrefix(f.Hash, h) {
- return f.Hash
+ if h != "" {
+ for _, f := range failures() {
+ if strings.HasPrefix(f.Hash, h) {
+ return f.Hash
+ }
}
}
log.Fatalf("invalid hash %q; failed to finds its full hash. Not a recent failure?", h)
panic("unreachable")
}
-// hash may be empty
+// wipe wipes the git hash failure for the provided failure.
+// Only the main go repo is currently supported.
func wipe(builder, hash string) {
- if hash != "" {
- log.Printf("Clearing %s, hash %s", builder, hash)
- } else {
- log.Printf("Clearing all builds for %s", builder)
- }
vals := url.Values{
"builder": {builder},
"hash": {hash},
diff --git a/maintner/maintnerd/apipb/api.pb.go b/maintner/maintnerd/apipb/api.pb.go
index e459641..82c2ea5 100644
--- a/maintner/maintnerd/apipb/api.pb.go
+++ b/maintner/maintnerd/apipb/api.pb.go
@@ -20,6 +20,10 @@
ListGoReleasesRequest
ListGoReleasesResponse
GoRelease
+ DashboardRequest
+ DashboardResponse
+ DashCommit
+ DashRepoHead
*/
package apipb
@@ -412,6 +416,224 @@
return ""
}
+type DashboardRequest struct {
+ // page is the zero-based page number.
+ // TODO: deprecate, replace with time or commit continuation token.
+ Page int32 `protobuf:"varint,1,opt,name=page" json:"page,omitempty"`
+ // repo is which repo to show ("go", "golang.org/x/net", "" means go).
+ Repo string `protobuf:"bytes,2,opt,name=repo" json:"repo,omitempty"`
+ // branch specifies which branch to show ("master", "release-branch.go1.13").
+ // Empty means "master".
+ // The special branch value "mixed" means to blend together all branches by commit time.
+ Branch string `protobuf:"bytes,3,opt,name=branch" json:"branch,omitempty"`
+ // max_commits specifies the number of commits that are desired.
+ // Zero means to use a default.
+ MaxCommits int32 `protobuf:"varint,4,opt,name=max_commits,json=maxCommits" json:"max_commits,omitempty"`
+}
+
+func (m *DashboardRequest) Reset() { *m = DashboardRequest{} }
+func (m *DashboardRequest) String() string { return proto.CompactTextString(m) }
+func (*DashboardRequest) ProtoMessage() {}
+func (*DashboardRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+
+func (m *DashboardRequest) GetPage() int32 {
+ if m != nil {
+ return m.Page
+ }
+ return 0
+}
+
+func (m *DashboardRequest) GetRepo() string {
+ if m != nil {
+ return m.Repo
+ }
+ return ""
+}
+
+func (m *DashboardRequest) GetBranch() string {
+ if m != nil {
+ return m.Branch
+ }
+ return ""
+}
+
+func (m *DashboardRequest) GetMaxCommits() int32 {
+ if m != nil {
+ return m.MaxCommits
+ }
+ return 0
+}
+
+type DashboardResponse struct {
+ // commits are the commits to display, starting with the newest.
+ Commits []*DashCommit `protobuf:"bytes,1,rep,name=commits" json:"commits,omitempty"`
+ // commits_truncated is whether the returned commits were truncated.
+ CommitsTruncated bool `protobuf:"varint,5,opt,name=commits_truncated,json=commitsTruncated" json:"commits_truncated,omitempty"`
+ // repo_heads contains the current head commit (of their master
+ // branch) for every repo on Go's Gerrit server.
+ RepoHeads []*DashRepoHead `protobuf:"bytes,2,rep,name=repo_heads,json=repoHeads" json:"repo_heads,omitempty"`
+ Branches []string `protobuf:"bytes,3,rep,name=branches" json:"branches,omitempty"`
+ // releases is the same content is ListGoReleasesResponse, but with the addition of a "master"
+ // release first, containing the info for the "master" branch, which is just commits[0]
+ // if page 0. But if page != 0, the master head wouldn't be
+ // available otherwise, so we denormalize it a bit here:
+ // It's sorted from newest to oldest (master, release-branch.go1.latest, release-branch.go1.prior)
+ // Only the branch_name and branch_commit fields are guaranteed to be populated.
+ Releases []*GoRelease `protobuf:"bytes,4,rep,name=releases" json:"releases,omitempty"`
+}
+
+func (m *DashboardResponse) Reset() { *m = DashboardResponse{} }
+func (m *DashboardResponse) String() string { return proto.CompactTextString(m) }
+func (*DashboardResponse) ProtoMessage() {}
+func (*DashboardResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+
+func (m *DashboardResponse) GetCommits() []*DashCommit {
+ if m != nil {
+ return m.Commits
+ }
+ return nil
+}
+
+func (m *DashboardResponse) GetCommitsTruncated() bool {
+ if m != nil {
+ return m.CommitsTruncated
+ }
+ return false
+}
+
+func (m *DashboardResponse) GetRepoHeads() []*DashRepoHead {
+ if m != nil {
+ return m.RepoHeads
+ }
+ return nil
+}
+
+func (m *DashboardResponse) GetBranches() []string {
+ if m != nil {
+ return m.Branches
+ }
+ return nil
+}
+
+func (m *DashboardResponse) GetReleases() []*GoRelease {
+ if m != nil {
+ return m.Releases
+ }
+ return nil
+}
+
+type DashCommit struct {
+ // commit is the git commit hash ("26957168c4c0cdcc7ca4f0b19d0eb19474d224ac").
+ Commit string `protobuf:"bytes,1,opt,name=commit" json:"commit,omitempty"`
+ // author_name is the git author name part ("Foo Bar").
+ AuthorName string `protobuf:"bytes,2,opt,name=author_name,json=authorName" json:"author_name,omitempty"`
+ // author_email is the git author email part ("foo@bar.com").
+ AuthorEmail string `protobuf:"bytes,3,opt,name=author_email,json=authorEmail" json:"author_email,omitempty"`
+ // commit_time_sec is the timestamp of git commit time, in unix seconds.
+ CommitTimeSec int64 `protobuf:"varint,4,opt,name=commit_time_sec,json=commitTimeSec" json:"commit_time_sec,omitempty"`
+ // title is the git commit's first line ("runtime: fix all the bugs").
+ Title string `protobuf:"bytes,5,opt,name=title" json:"title,omitempty"`
+ // branch is the branch this commit was queried from ("master", "release-branch.go1.14")/
+ // This is normally redundant but is useful when DashboardRequest.branch == "mixed".
+ Branch string `protobuf:"bytes,7,opt,name=branch" json:"branch,omitempty"`
+ // For non-go repos, go_commit_at_time is what the Go master commit was at
+ // the time of DashCommit.commit_time.
+ GoCommitAtTime string `protobuf:"bytes,6,opt,name=go_commit_at_time,json=goCommitAtTime" json:"go_commit_at_time,omitempty"`
+ // For non-go repos, go_commit_latest is the most recent Go master commit that's
+ // older than the the following x/foo commit's commit_time.
+ // If DashCommit is the current HEAD, go_commit_at_time can continue to update.
+ // go_commit_at_time might be the same as go_commit_at_time.
+ GoCommitLatest string `protobuf:"bytes,8,opt,name=go_commit_latest,json=goCommitLatest" json:"go_commit_latest,omitempty"`
+}
+
+func (m *DashCommit) Reset() { *m = DashCommit{} }
+func (m *DashCommit) String() string { return proto.CompactTextString(m) }
+func (*DashCommit) ProtoMessage() {}
+func (*DashCommit) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+func (m *DashCommit) GetCommit() string {
+ if m != nil {
+ return m.Commit
+ }
+ return ""
+}
+
+func (m *DashCommit) GetAuthorName() string {
+ if m != nil {
+ return m.AuthorName
+ }
+ return ""
+}
+
+func (m *DashCommit) GetAuthorEmail() string {
+ if m != nil {
+ return m.AuthorEmail
+ }
+ return ""
+}
+
+func (m *DashCommit) GetCommitTimeSec() int64 {
+ if m != nil {
+ return m.CommitTimeSec
+ }
+ return 0
+}
+
+func (m *DashCommit) GetTitle() string {
+ if m != nil {
+ return m.Title
+ }
+ return ""
+}
+
+func (m *DashCommit) GetBranch() string {
+ if m != nil {
+ return m.Branch
+ }
+ return ""
+}
+
+func (m *DashCommit) GetGoCommitAtTime() string {
+ if m != nil {
+ return m.GoCommitAtTime
+ }
+ return ""
+}
+
+func (m *DashCommit) GetGoCommitLatest() string {
+ if m != nil {
+ return m.GoCommitLatest
+ }
+ return ""
+}
+
+type DashRepoHead struct {
+ // gerrit_project is Gerrit project name ("net", "go").
+ GerritProject string `protobuf:"bytes,1,opt,name=gerrit_project,json=gerritProject" json:"gerrit_project,omitempty"`
+ // commit is the current top-level commit in that project.
+ // (currently always on the master branch)
+ Commit *DashCommit `protobuf:"bytes,2,opt,name=commit" json:"commit,omitempty"`
+}
+
+func (m *DashRepoHead) Reset() { *m = DashRepoHead{} }
+func (m *DashRepoHead) String() string { return proto.CompactTextString(m) }
+func (*DashRepoHead) ProtoMessage() {}
+func (*DashRepoHead) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
+func (m *DashRepoHead) GetGerritProject() string {
+ if m != nil {
+ return m.GerritProject
+ }
+ return ""
+}
+
+func (m *DashRepoHead) GetCommit() *DashCommit {
+ if m != nil {
+ return m.Commit
+ }
+ return nil
+}
+
func init() {
proto.RegisterType((*HasAncestorRequest)(nil), "apipb.HasAncestorRequest")
proto.RegisterType((*HasAncestorResponse)(nil), "apipb.HasAncestorResponse")
@@ -425,6 +647,10 @@
proto.RegisterType((*ListGoReleasesRequest)(nil), "apipb.ListGoReleasesRequest")
proto.RegisterType((*ListGoReleasesResponse)(nil), "apipb.ListGoReleasesResponse")
proto.RegisterType((*GoRelease)(nil), "apipb.GoRelease")
+ proto.RegisterType((*DashboardRequest)(nil), "apipb.DashboardRequest")
+ proto.RegisterType((*DashboardResponse)(nil), "apipb.DashboardResponse")
+ proto.RegisterType((*DashCommit)(nil), "apipb.DashCommit")
+ proto.RegisterType((*DashRepoHead)(nil), "apipb.DashRepoHead")
}
// Reference imports to suppress errors if they are not otherwise used.
@@ -456,6 +682,11 @@
// The response is guaranteed to have two versions, otherwise an error
// is returned.
ListGoReleases(ctx context.Context, in *ListGoReleasesRequest, opts ...grpc.CallOption) (*ListGoReleasesResponse, error)
+ // GetDashboard returns the information for the build.golang.org
+ // dashboard. It does not (at least currently)
+ // contain any pass/fail information; it only contains information on the branches
+ // and commits themselves.
+ GetDashboard(ctx context.Context, in *DashboardRequest, opts ...grpc.CallOption) (*DashboardResponse, error)
}
type maintnerServiceClient struct {
@@ -502,6 +733,15 @@
return out, nil
}
+func (c *maintnerServiceClient) GetDashboard(ctx context.Context, in *DashboardRequest, opts ...grpc.CallOption) (*DashboardResponse, error) {
+ out := new(DashboardResponse)
+ err := grpc.Invoke(ctx, "/apipb.MaintnerService/GetDashboard", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
// Server API for MaintnerService service
type MaintnerServiceServer interface {
@@ -523,6 +763,11 @@
// The response is guaranteed to have two versions, otherwise an error
// is returned.
ListGoReleases(context.Context, *ListGoReleasesRequest) (*ListGoReleasesResponse, error)
+ // GetDashboard returns the information for the build.golang.org
+ // dashboard. It does not (at least currently)
+ // contain any pass/fail information; it only contains information on the branches
+ // and commits themselves.
+ GetDashboard(context.Context, *DashboardRequest) (*DashboardResponse, error)
}
func RegisterMaintnerServiceServer(s *grpc.Server, srv MaintnerServiceServer) {
@@ -601,6 +846,24 @@
return interceptor(ctx, in, info, handler)
}
+func _MaintnerService_GetDashboard_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(DashboardRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(MaintnerServiceServer).GetDashboard(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/apipb.MaintnerService/GetDashboard",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(MaintnerServiceServer).GetDashboard(ctx, req.(*DashboardRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
var _MaintnerService_serviceDesc = grpc.ServiceDesc{
ServiceName: "apipb.MaintnerService",
HandlerType: (*MaintnerServiceServer)(nil),
@@ -621,6 +884,10 @@
MethodName: "ListGoReleases",
Handler: _MaintnerService_ListGoReleases_Handler,
},
+ {
+ MethodName: "GetDashboard",
+ Handler: _MaintnerService_GetDashboard_Handler,
+ },
},
Streams: []grpc.StreamDesc{},
Metadata: "api.proto",
@@ -629,49 +896,67 @@
func init() { proto.RegisterFile("api.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 700 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdb, 0x4e, 0x1b, 0x31,
- 0x10, 0x55, 0x92, 0xe6, 0x36, 0x21, 0x29, 0xb8, 0x84, 0x2e, 0xa1, 0x08, 0xba, 0xa8, 0x15, 0x0f,
- 0x15, 0xaa, 0xa8, 0x7a, 0x79, 0xed, 0x45, 0x04, 0xda, 0xa6, 0xaa, 0x16, 0x44, 0x1f, 0x57, 0x26,
- 0x71, 0x36, 0x0b, 0xac, 0xbd, 0xb5, 0x1d, 0x50, 0x3e, 0xa9, 0x7f, 0xd0, 0x9f, 0xe8, 0x3f, 0x55,
- 0xb6, 0xc7, 0x4b, 0x02, 0xf4, 0xa1, 0x6f, 0x3b, 0xe7, 0xcc, 0x1c, 0x8f, 0xc7, 0xc7, 0x5e, 0x68,
- 0xd2, 0x3c, 0xdd, 0xcb, 0xa5, 0xd0, 0x82, 0x54, 0x69, 0x9e, 0xe6, 0x67, 0xe1, 0x21, 0x90, 0x43,
- 0xaa, 0xde, 0xf3, 0x21, 0x53, 0x5a, 0xc8, 0x88, 0xfd, 0x9c, 0x32, 0xa5, 0xc9, 0x1a, 0xd4, 0x86,
- 0x22, 0xcb, 0x52, 0x1d, 0x94, 0xb6, 0x4b, 0xbb, 0xcd, 0x08, 0x23, 0xd2, 0x83, 0x06, 0xc5, 0xd4,
- 0xa0, 0x6c, 0x99, 0x22, 0x0e, 0x63, 0x78, 0xb4, 0xa0, 0xa4, 0x72, 0xc1, 0x15, 0x23, 0x4f, 0x61,
- 0x69, 0x42, 0x55, 0x5c, 0x94, 0x19, 0xc1, 0x46, 0xd4, 0x9a, 0xdc, 0xa4, 0x92, 0x67, 0xd0, 0x99,
- 0xf2, 0x0b, 0x2e, 0xae, 0x79, 0x8c, 0xab, 0x96, 0x6d, 0x52, 0x1b, 0xd1, 0x8f, 0x16, 0x0c, 0x33,
- 0x68, 0xf7, 0x99, 0x8e, 0xd8, 0xd8, 0x77, 0xb9, 0x0c, 0x15, 0xc9, 0xc6, 0xd8, 0xa2, 0xf9, 0x24,
- 0x3b, 0xd0, 0x4e, 0x98, 0x94, 0xa9, 0x8e, 0x15, 0x93, 0x57, 0xcc, 0x37, 0xb9, 0xe4, 0xc0, 0x63,
- 0x8b, 0x99, 0xe5, 0x30, 0x29, 0x97, 0xe2, 0x9c, 0x0d, 0x75, 0x50, 0xb1, 0x59, 0x58, 0xfa, 0xdd,
- 0x81, 0xe1, 0x73, 0xe8, 0xf8, 0xe5, 0x70, 0x2b, 0xab, 0x50, 0xbd, 0xa2, 0x97, 0x53, 0x86, 0x2b,
- 0xba, 0x20, 0x7c, 0x0b, 0xab, 0x7d, 0x71, 0x90, 0xf2, 0xd1, 0x89, 0x9c, 0xfd, 0x10, 0xf2, 0xc2,
- 0x77, 0xb7, 0x05, 0xad, 0xb1, 0x90, 0xb1, 0xd2, 0x34, 0x49, 0x79, 0x82, 0xfb, 0x86, 0xb1, 0x90,
- 0xc7, 0x0e, 0x09, 0xbf, 0x40, 0xf7, 0x56, 0x21, 0xae, 0xb3, 0x0f, 0xf5, 0x6b, 0x9a, 0x6a, 0x57,
- 0x55, 0xd9, 0x6d, 0xed, 0x07, 0x7b, 0xf6, 0xb0, 0xf6, 0xfa, 0xb6, 0x41, 0x4c, 0x3f, 0xd2, 0x2c,
- 0x8b, 0x7c, 0x62, 0xf8, 0xbb, 0x0c, 0x2b, 0x77, 0x68, 0x12, 0x40, 0xdd, 0xef, 0xd1, 0xf5, 0xec,
- 0x43, 0x73, 0xc2, 0x67, 0x92, 0xf2, 0xe1, 0x04, 0x47, 0x84, 0x11, 0xd9, 0x80, 0xe6, 0x70, 0x42,
- 0x79, 0xc2, 0xe2, 0x74, 0x84, 0x73, 0x69, 0x38, 0xe0, 0x68, 0x34, 0x67, 0x8b, 0x07, 0x0b, 0xb6,
- 0x08, 0xa0, 0x7e, 0xc5, 0xa4, 0x4a, 0x05, 0x0f, 0x9a, 0xdb, 0xa5, 0xdd, 0x6a, 0xe4, 0x43, 0x23,
- 0x97, 0x08, 0x7f, 0xaa, 0xd5, 0xed, 0x8a, 0x91, 0x4b, 0x84, 0x3b, 0x50, 0x24, 0xb1, 0x8d, 0x9a,
- 0x27, 0x3f, 0xb8, 0x46, 0x5e, 0x02, 0x24, 0x22, 0xf6, 0xb2, 0x75, 0x3b, 0x87, 0x15, 0x9c, 0xc3,
- 0x80, 0x9e, 0x0b, 0x39, 0x48, 0xb9, 0x90, 0x51, 0x33, 0x11, 0xa7, 0xb8, 0xd6, 0x1b, 0x68, 0x69,
- 0x39, 0x8b, 0x33, 0xa6, 0x14, 0x4d, 0x58, 0xd0, 0xb0, 0x25, 0x5d, 0x2c, 0x39, 0x91, 0xb3, 0x53,
- 0xa1, 0xd9, 0xc0, 0x91, 0x11, 0x68, 0x39, 0xc3, 0xef, 0x90, 0x42, 0x67, 0x91, 0x35, 0xfb, 0xf1,
- 0x2a, 0x38, 0x36, 0x0c, 0x4d, 0xcb, 0x74, 0xaa, 0x27, 0x42, 0x9a, 0xf1, 0x98, 0xc9, 0x55, 0xa2,
- 0x86, 0x03, 0x8e, 0x46, 0xf3, 0x63, 0xa8, 0x2c, 0x8c, 0x21, 0x7c, 0x07, 0x70, 0xd3, 0xb3, 0xf1,
- 0x51, 0x66, 0x22, 0x2b, 0x5e, 0x8d, 0x5c, 0x60, 0x51, 0x43, 0x5b, 0x59, 0x83, 0x9a, 0x20, 0x7c,
- 0x0c, 0xdd, 0xaf, 0xa9, 0xd2, 0x7d, 0x11, 0xb1, 0x4b, 0x46, 0x15, 0x53, 0x68, 0xaf, 0xf0, 0x00,
- 0xd6, 0x6e, 0x13, 0x68, 0x9f, 0x17, 0xd0, 0x90, 0x88, 0xa1, 0x7f, 0x96, 0xbd, 0x7f, 0x7c, 0x72,
- 0x54, 0x64, 0x84, 0x7f, 0x4a, 0xd0, 0x2c, 0xf0, 0xff, 0x69, 0xcd, 0xa0, 0x39, 0xd5, 0xc3, 0x09,
- 0x6e, 0xd6, 0x05, 0x64, 0x1d, 0x1a, 0x9a, 0x26, 0x31, 0xa7, 0x19, 0x43, 0x97, 0xd4, 0x35, 0x4d,
- 0xbe, 0xd1, 0x8c, 0x91, 0x4d, 0x00, 0x43, 0x15, 0x6e, 0x30, 0x64, 0x53, 0xd3, 0x04, 0xed, 0xb0,
- 0x05, 0x2d, 0xe7, 0x05, 0x57, 0x5c, 0xb3, 0x3c, 0x38, 0xc8, 0xd6, 0xef, 0x40, 0x1b, 0x13, 0x50,
- 0xa2, 0xee, 0x6e, 0xb7, 0x03, 0x9d, 0xca, 0xfe, 0xaf, 0x32, 0x3c, 0x1c, 0xd0, 0x94, 0x6b, 0xce,
- 0xa4, 0xb9, 0xf0, 0xe9, 0x90, 0x91, 0x4f, 0xd0, 0x9a, 0x7b, 0x9a, 0xc8, 0x3a, 0x8e, 0xe3, 0xee,
- 0xc3, 0xd7, 0xeb, 0xdd, 0x47, 0xe1, 0x5c, 0x5f, 0x43, 0xcd, 0x3d, 0x08, 0x64, 0xb5, 0xb8, 0x8f,
- 0x73, 0xcf, 0x51, 0xaf, 0x7b, 0x0b, 0xc5, 0xb2, 0xcf, 0xd0, 0x5e, 0xb8, 0xe6, 0x64, 0xa3, 0x38,
- 0x8d, 0xbb, 0xaf, 0x46, 0xef, 0xc9, 0xfd, 0x24, 0x6a, 0x0d, 0xa0, 0xb3, 0x78, 0xe8, 0xc4, 0xe7,
- 0xdf, 0x6b, 0x92, 0xde, 0xe6, 0x3f, 0x58, 0x27, 0x77, 0x56, 0xb3, 0xbf, 0x82, 0x57, 0x7f, 0x03,
- 0x00, 0x00, 0xff, 0xff, 0x0d, 0x79, 0xa6, 0xa3, 0x17, 0x06, 0x00, 0x00,
+ // 988 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0xed, 0x6e, 0xdb, 0x36,
+ 0x17, 0x86, 0xed, 0x38, 0xb6, 0x8f, 0xe3, 0x34, 0x61, 0x93, 0x56, 0x75, 0xdf, 0x22, 0xa9, 0x8a,
+ 0xb7, 0x48, 0xd1, 0x21, 0x18, 0x32, 0xec, 0xe3, 0x6f, 0xb6, 0xae, 0x4e, 0xb6, 0x7a, 0x18, 0x94,
+ 0xa0, 0xfb, 0xa9, 0x31, 0xf2, 0x89, 0xac, 0xd6, 0x12, 0x35, 0x92, 0x4e, 0x9b, 0xeb, 0xd8, 0xcd,
+ 0xec, 0x26, 0x76, 0x25, 0xdb, 0x45, 0x0c, 0x24, 0x0f, 0x15, 0x29, 0x71, 0x31, 0xec, 0x9f, 0xce,
+ 0x73, 0x3e, 0x79, 0xce, 0x43, 0x1e, 0xc1, 0x80, 0x97, 0xd9, 0x61, 0x29, 0x85, 0x16, 0xac, 0xcb,
+ 0xcb, 0xac, 0xbc, 0x08, 0x4f, 0x80, 0x9d, 0x70, 0x75, 0x5c, 0x24, 0xa8, 0xb4, 0x90, 0x11, 0xfe,
+ 0xb6, 0x44, 0xa5, 0xd9, 0x03, 0x58, 0x4f, 0x44, 0x9e, 0x67, 0x3a, 0x68, 0xed, 0xb7, 0x0e, 0x06,
+ 0x11, 0x49, 0x6c, 0x0c, 0x7d, 0x4e, 0xa6, 0x41, 0xdb, 0x6a, 0x2a, 0x39, 0x8c, 0xe1, 0x7e, 0x23,
+ 0x92, 0x2a, 0x45, 0xa1, 0x90, 0x3d, 0x85, 0x8d, 0x39, 0x57, 0x71, 0xe5, 0x66, 0x02, 0xf6, 0xa3,
+ 0xe1, 0xfc, 0xc6, 0x94, 0xfd, 0x1f, 0x36, 0x97, 0xc5, 0xfb, 0x42, 0x7c, 0x28, 0x62, 0xca, 0xda,
+ 0xb6, 0x46, 0x23, 0x42, 0xbf, 0xb3, 0x60, 0x98, 0xc3, 0x68, 0x82, 0x3a, 0xc2, 0x4b, 0x5f, 0xe5,
+ 0x16, 0x74, 0x24, 0x5e, 0x52, 0x89, 0xe6, 0x93, 0x3d, 0x83, 0x51, 0x8a, 0x52, 0x66, 0x3a, 0x56,
+ 0x28, 0xaf, 0xd0, 0x17, 0xb9, 0xe1, 0xc0, 0x33, 0x8b, 0x99, 0x74, 0x64, 0x54, 0x4a, 0xf1, 0x0e,
+ 0x13, 0x1d, 0x74, 0xac, 0x15, 0xb9, 0xfe, 0xec, 0xc0, 0xf0, 0x39, 0x6c, 0xfa, 0x74, 0x74, 0x94,
+ 0x1d, 0xe8, 0x5e, 0xf1, 0xc5, 0x12, 0x29, 0xa3, 0x13, 0xc2, 0xaf, 0x61, 0x67, 0x22, 0x5e, 0x67,
+ 0xc5, 0xec, 0x5c, 0x5e, 0xff, 0x22, 0xe4, 0x7b, 0x5f, 0xdd, 0x1e, 0x0c, 0x2f, 0x85, 0x8c, 0x95,
+ 0xe6, 0x69, 0x56, 0xa4, 0x74, 0x6e, 0xb8, 0x14, 0xf2, 0xcc, 0x21, 0xe1, 0x8f, 0xb0, 0x7b, 0xcb,
+ 0x91, 0xf2, 0x1c, 0x41, 0xef, 0x03, 0xcf, 0xb4, 0xf3, 0xea, 0x1c, 0x0c, 0x8f, 0x82, 0x43, 0x3b,
+ 0xac, 0xc3, 0x89, 0x2d, 0x90, 0xcc, 0x4f, 0x35, 0xe6, 0x91, 0x37, 0x0c, 0xff, 0x68, 0xc3, 0xf6,
+ 0x1d, 0x35, 0x0b, 0xa0, 0xe7, 0xcf, 0xe8, 0x6a, 0xf6, 0xa2, 0x99, 0xf0, 0x85, 0xe4, 0x45, 0x32,
+ 0xa7, 0x16, 0x91, 0xc4, 0x1e, 0xc3, 0x20, 0x99, 0xf3, 0x22, 0xc5, 0x38, 0x9b, 0x51, 0x5f, 0xfa,
+ 0x0e, 0x38, 0x9d, 0xd5, 0x68, 0xb1, 0xd6, 0xa0, 0x45, 0x00, 0xbd, 0x2b, 0x94, 0x2a, 0x13, 0x45,
+ 0x30, 0xd8, 0x6f, 0x1d, 0x74, 0x23, 0x2f, 0x9a, 0x70, 0xa9, 0xf0, 0x53, 0xed, 0xee, 0x77, 0x4c,
+ 0xb8, 0x54, 0xb8, 0x81, 0x92, 0x92, 0xca, 0x58, 0xf7, 0xca, 0x6f, 0x5d, 0x21, 0x9f, 0x03, 0xa4,
+ 0x22, 0xf6, 0x61, 0x7b, 0xb6, 0x0f, 0xdb, 0xd4, 0x87, 0x29, 0x7f, 0x27, 0xe4, 0x34, 0x2b, 0x84,
+ 0x8c, 0x06, 0xa9, 0x78, 0x4b, 0xb9, 0xbe, 0x82, 0xa1, 0x96, 0xd7, 0x71, 0x8e, 0x4a, 0xf1, 0x14,
+ 0x83, 0xbe, 0x75, 0xd9, 0x25, 0x97, 0x73, 0x79, 0xfd, 0x56, 0x68, 0x9c, 0x3a, 0x65, 0x04, 0x5a,
+ 0x5e, 0xd3, 0x77, 0xc8, 0x61, 0xb3, 0xa9, 0x35, 0xe7, 0xf1, 0x51, 0xa8, 0x6d, 0x24, 0x9a, 0x92,
+ 0xf9, 0x52, 0xcf, 0x85, 0x34, 0xed, 0x31, 0x9d, 0xeb, 0x44, 0x7d, 0x07, 0x9c, 0xce, 0xea, 0x6d,
+ 0xe8, 0x34, 0xda, 0x10, 0x7e, 0x03, 0x70, 0x53, 0xb3, 0xe1, 0x51, 0x6e, 0x24, 0x1b, 0xbc, 0x1b,
+ 0x39, 0xc1, 0xa2, 0x46, 0x6d, 0xc3, 0x1a, 0xd4, 0x08, 0xe1, 0x43, 0xd8, 0x7d, 0x93, 0x29, 0x3d,
+ 0x11, 0x11, 0x2e, 0x90, 0x2b, 0x54, 0x44, 0xaf, 0xf0, 0x35, 0x3c, 0xb8, 0xad, 0x20, 0xfa, 0x7c,
+ 0x06, 0x7d, 0x49, 0x18, 0xf1, 0x67, 0xcb, 0xf3, 0xc7, 0x1b, 0x47, 0x95, 0x45, 0xf8, 0x67, 0x0b,
+ 0x06, 0x15, 0xfe, 0x5f, 0x4a, 0x33, 0x68, 0xc9, 0x75, 0x32, 0xa7, 0xc3, 0x3a, 0x81, 0x3d, 0x82,
+ 0xbe, 0xe6, 0x69, 0x5c, 0xf0, 0x1c, 0x89, 0x25, 0x3d, 0xcd, 0xd3, 0x9f, 0x78, 0x8e, 0xec, 0x09,
+ 0x80, 0x51, 0x55, 0x6c, 0x30, 0xca, 0x81, 0xe6, 0x29, 0xd1, 0x61, 0x0f, 0x86, 0x8e, 0x0b, 0xce,
+ 0x79, 0xdd, 0xea, 0xc1, 0x41, 0xd6, 0xff, 0x19, 0x8c, 0xc8, 0x80, 0x42, 0xf4, 0xdc, 0xed, 0x76,
+ 0x20, 0xbd, 0x12, 0x0a, 0xb6, 0x5e, 0x71, 0x35, 0xbf, 0x10, 0x5c, 0xce, 0xfc, 0x55, 0x64, 0xb0,
+ 0x56, 0xfa, 0x61, 0x76, 0x23, 0xfb, 0x6d, 0x30, 0x89, 0xa5, 0x20, 0xfa, 0xdb, 0xef, 0xda, 0xa5,
+ 0xe8, 0x34, 0x2e, 0xc5, 0x1e, 0x0c, 0x73, 0xfe, 0x91, 0xb2, 0x2a, 0x7b, 0xac, 0x6e, 0x04, 0x39,
+ 0xff, 0xe8, 0x72, 0xaa, 0xf0, 0xaf, 0x16, 0x6c, 0xd7, 0xb2, 0xd2, 0x20, 0x5e, 0x42, 0xcf, 0xbb,
+ 0xb4, 0x1a, 0xfc, 0x35, 0xa6, 0xce, 0x35, 0xf2, 0x16, 0xec, 0x25, 0x6c, 0xd3, 0x67, 0xac, 0xe5,
+ 0xb2, 0x48, 0xb8, 0xc6, 0x99, 0xed, 0x51, 0x3f, 0xda, 0x22, 0xc5, 0xb9, 0xc7, 0xd9, 0x11, 0x80,
+ 0x29, 0x38, 0x9e, 0x23, 0x9f, 0xa9, 0xa0, 0x6d, 0x83, 0xdf, 0xaf, 0x05, 0x8f, 0xb0, 0x14, 0x27,
+ 0xc8, 0x67, 0xd1, 0x40, 0xd2, 0x97, 0x32, 0x6f, 0xb7, 0x3b, 0x0e, 0xaa, 0xa0, 0xe3, 0x2e, 0x9b,
+ 0x97, 0x1b, 0x94, 0x59, 0xfb, 0x57, 0xca, 0xfc, 0xde, 0x06, 0xb8, 0x39, 0xc2, 0x27, 0x97, 0xc5,
+ 0x1e, 0x0c, 0xe9, 0xae, 0xd8, 0x79, 0xba, 0x46, 0x83, 0x83, 0xec, 0x3c, 0x9f, 0xc2, 0x06, 0x19,
+ 0x60, 0xce, 0xb3, 0x05, 0x35, 0x9d, 0x9c, 0xbe, 0x37, 0x10, 0x7b, 0x0e, 0xf7, 0x5c, 0xb4, 0x58,
+ 0x67, 0x39, 0xc6, 0x0a, 0x13, 0xdb, 0xfd, 0x4e, 0x34, 0x72, 0xf0, 0x79, 0x96, 0xe3, 0x19, 0x26,
+ 0x86, 0x8b, 0x3a, 0xd3, 0x0b, 0x24, 0x56, 0x39, 0xa1, 0x36, 0xcf, 0x5e, 0x63, 0x9e, 0x2f, 0x60,
+ 0xbb, 0x7a, 0x95, 0x62, 0xee, 0x62, 0x13, 0xdf, 0x36, 0xfd, 0xeb, 0x74, 0x6c, 0x63, 0xb3, 0x03,
+ 0xd8, 0xba, 0x31, 0x5d, 0x70, 0x8d, 0x4a, 0x07, 0xfd, 0xa6, 0xe5, 0x1b, 0x8b, 0x86, 0xbf, 0xc2,
+ 0x46, 0xbd, 0xf5, 0x2b, 0xd6, 0x4c, 0x6b, 0xc5, 0x9a, 0x61, 0x2f, 0xaa, 0xee, 0x99, 0x06, 0xad,
+ 0xe4, 0x08, 0x19, 0x1c, 0xfd, 0xdd, 0x86, 0x7b, 0x53, 0x9e, 0x15, 0xba, 0x40, 0x69, 0x76, 0x59,
+ 0x96, 0x20, 0x7b, 0x05, 0xc3, 0xda, 0xd6, 0x65, 0x8f, 0xc8, 0xfb, 0xee, 0x4e, 0x1f, 0x8f, 0x57,
+ 0xa9, 0x88, 0xa9, 0x5f, 0xc2, 0xba, 0xdb, 0x75, 0x6c, 0xa7, 0x5a, 0x35, 0xb5, 0x4d, 0x3b, 0xde,
+ 0xbd, 0x85, 0x92, 0xdb, 0x0f, 0x30, 0x6a, 0x6c, 0x30, 0xf6, 0xb8, 0x62, 0xcd, 0xdd, 0x85, 0x38,
+ 0xfe, 0xdf, 0x6a, 0x25, 0xc5, 0x9a, 0xc2, 0x66, 0xf3, 0x3d, 0x63, 0xde, 0x7e, 0xe5, 0xfb, 0x37,
+ 0x7e, 0xf2, 0x09, 0x2d, 0x85, 0x3b, 0x86, 0x8d, 0x09, 0xea, 0xea, 0x4e, 0xb2, 0x87, 0xb5, 0xb6,
+ 0xd6, 0xdf, 0x86, 0x71, 0x70, 0x57, 0xe1, 0x42, 0x5c, 0xac, 0xdb, 0x1f, 0xa5, 0x2f, 0xfe, 0x09,
+ 0x00, 0x00, 0xff, 0xff, 0xa8, 0x18, 0x53, 0x68, 0x35, 0x09, 0x00, 0x00,
}
diff --git a/maintner/maintnerd/apipb/api.proto b/maintner/maintnerd/apipb/api.proto
index 6ca8c3a..577b80c 100644
--- a/maintner/maintnerd/apipb/api.proto
+++ b/maintner/maintnerd/apipb/api.proto
@@ -101,6 +101,86 @@
string branch_commit = 7; // most recent commit on the release branch, e.g., "edb6c16b9b62ed8586d2e3e422911d646095b7e5"
}
+message DashboardRequest {
+ // page is the zero-based page number.
+ // TODO: deprecate, replace with time or commit continuation token.
+ int32 page = 1;
+
+ // repo is which repo to show ("go", "golang.org/x/net", "" means go).
+ string repo = 2;
+
+ // branch specifies which branch to show ("master", "release-branch.go1.13").
+ // Empty means "master".
+ // The special branch value "mixed" means to blend together all branches by commit time.
+ string branch = 3;
+
+ // max_commits specifies the number of commits that are desired.
+ // Zero means to use a default.
+ int32 max_commits = 4;
+}
+
+message DashboardResponse {
+ // commits are the commits to display, starting with the newest.
+ repeated DashCommit commits = 1;
+
+ // commits_truncated is whether the returned commits were truncated.
+ bool commits_truncated = 5;
+
+ // repo_heads contains the current head commit (of their master
+ // branch) for every repo on Go's Gerrit server.
+ repeated DashRepoHead repo_heads = 2;
+
+ repeated string branches = 3;
+
+ // releases is the same content is ListGoReleasesResponse, but with the addition of a "master"
+ // release first, containing the info for the "master" branch, which is just commits[0]
+ // if page 0. But if page != 0, the master head wouldn't be
+ // available otherwise, so we denormalize it a bit here:
+ // It's sorted from newest to oldest (master, release-branch.go1.latest, release-branch.go1.prior)
+ // Only the branch_name and branch_commit fields are guaranteed to be populated.
+ repeated GoRelease releases = 4;
+}
+
+message DashCommit {
+ // commit is the git commit hash ("26957168c4c0cdcc7ca4f0b19d0eb19474d224ac").
+ string commit = 1;
+
+ // author_name is the git author name part ("Foo Bar").
+ string author_name = 2; // "Foo Bar"
+
+ // author_email is the git author email part ("foo@bar.com").
+ string author_email = 3; // "foo@bar.com"
+
+ // commit_time_sec is the timestamp of git commit time, in unix seconds.
+ int64 commit_time_sec = 4;
+
+ // title is the git commit's first line ("runtime: fix all the bugs").
+ string title = 5;
+
+ // branch is the branch this commit was queried from ("master", "release-branch.go1.14")/
+ // This is normally redundant but is useful when DashboardRequest.branch == "mixed".
+ string branch = 7;
+
+ // For non-go repos, go_commit_at_time is what the Go master commit was at
+ // the time of DashCommit.commit_time.
+ string go_commit_at_time = 6;
+
+ // For non-go repos, go_commit_latest is the most recent Go master commit that's
+ // older than the the following x/foo commit's commit_time.
+ // If DashCommit is the current HEAD, go_commit_at_time can continue to update.
+ // go_commit_at_time might be the same as go_commit_at_time.
+ string go_commit_latest = 8;
+}
+
+message DashRepoHead {
+ // gerrit_project is Gerrit project name ("net", "go").
+ string gerrit_project = 1;
+
+ // commit is the current top-level commit in that project.
+ // (currently always on the master branch)
+ DashCommit commit = 2;
+}
+
service MaintnerService {
// HasAncestor reports whether one commit contains another commit
// in its git history.
@@ -125,4 +205,10 @@
// The response is guaranteed to have two versions, otherwise an error
// is returned.
rpc ListGoReleases(ListGoReleasesRequest) returns (ListGoReleasesResponse);
+
+ // GetDashboard returns the information for the build.golang.org
+ // dashboard. It does not (at least currently)
+ // contain any pass/fail information; it only contains information on the branches
+ // and commits themselves.
+ rpc GetDashboard(DashboardRequest) returns (DashboardResponse);
}
diff --git a/maintner/maintnerd/maintapi/api.go b/maintner/maintnerd/maintapi/api.go
index 662a73a..a5c2185 100644
--- a/maintner/maintnerd/maintapi/api.go
+++ b/maintner/maintnerd/maintapi/api.go
@@ -21,6 +21,9 @@
"golang.org/x/build/maintner"
"golang.org/x/build/maintner/maintnerd/apipb"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
+ "golang.org/x/build/repos"
+ "grpc.go4.org"
+ "grpc.go4.org/codes"
)
// NewAPIService creates a gRPC Server that serves the Maintner API for the given corpus.
@@ -445,3 +448,334 @@
}
return rs[:2], nil
}
+
+func (s apiService) GetDashboard(ctx context.Context, req *apipb.DashboardRequest) (*apipb.DashboardResponse, error) {
+ s.c.RLock()
+ defer s.c.RUnlock()
+
+ res := new(apipb.DashboardResponse)
+ goProj := s.c.Gerrit().Project("go.googlesource.com", "go")
+ if goProj == nil {
+ // Return a normal error here, without grpc code
+ // NotFound, because we expect to find this.
+ return nil, errors.New("go gerrit project not found")
+ }
+ if req.Repo == "" {
+ req.Repo = "go"
+ }
+ projName, err := dashRepoToGerritProj(req.Repo)
+ if err != nil {
+ return nil, err
+ }
+ proj := s.c.Gerrit().Project("go.googlesource.com", projName)
+ if proj == nil {
+ return nil, grpc.Errorf(codes.NotFound, "repo project %q not found", projName)
+ }
+
+ // Populate res.Branches.
+ const headPrefix = "refs/heads/"
+ refHash := map[string]string{} // "master" -> git commit hash
+ goProj.ForeachNonChangeRef(func(ref string, hash maintner.GitHash) error {
+ if !strings.HasPrefix(ref, headPrefix) {
+ return nil
+ }
+ branch := strings.TrimPrefix(ref, headPrefix)
+ refHash[branch] = hash.String()
+ res.Branches = append(res.Branches, branch)
+ return nil
+ })
+
+ if req.Branch == "" {
+ req.Branch = "master"
+ }
+ branch := req.Branch
+ mixBranches := branch == "mixed" // mix all branches together, by commit time
+ if !mixBranches && refHash[branch] == "" {
+ return nil, grpc.Errorf(codes.NotFound, "unknown branch %q", branch)
+ }
+
+ commitsPerPage := int(req.MaxCommits)
+ if commitsPerPage < 0 {
+ return nil, grpc.Errorf(codes.InvalidArgument, "negative max commits")
+ }
+ if commitsPerPage > 1000 {
+ commitsPerPage = 1000
+ }
+ if commitsPerPage == 0 {
+ if mixBranches {
+ commitsPerPage = 500
+ } else {
+ commitsPerPage = 30 // what build.golang.org historically used
+ }
+ }
+
+ if req.Page < 0 {
+ return nil, grpc.Errorf(codes.InvalidArgument, "invalid page")
+ }
+ if req.Page != 0 && mixBranches {
+ return nil, grpc.Errorf(codes.InvalidArgument, "branch=mixed does not support pagination")
+ }
+ skip := int(req.Page) * commitsPerPage
+ if skip >= 10000 {
+ return nil, grpc.Errorf(codes.InvalidArgument, "too far back") // arbitrary
+ }
+
+ // Find branches to merge together.
+ //
+ // By default we only have one branch (the one the user
+ // specified). But in mixed mode, as used by the coordinator
+ // when trying to find work to do, we merge all the branches
+ // together into one timeline.
+ branches := []string{branch}
+ if mixBranches {
+ branches = res.Branches
+ }
+ var oldestSkipped time.Time
+ res.Commits, res.CommitsTruncated, oldestSkipped = s.listDashCommits(proj, branches, commitsPerPage, skip)
+
+ // For non-go repos, populate the Go commits that corresponding to each commit.
+ if projName != "go" {
+ s.addGoCommits(oldestSkipped, res.Commits)
+ }
+
+ // Populate res.RepoHeads: each Gerrit repo with what its
+ // current master ref is at.
+ res.RepoHeads = s.dashRepoHeads()
+
+ // Populate res.Releases (the currently supported releases)
+ // with "master" followed by the past two release branches.
+ res.Releases = append(res.Releases, &apipb.GoRelease{
+ BranchName: "master",
+ BranchCommit: refHash["master"],
+ })
+ releases, err := supportedGoReleases(goProj)
+ if err != nil {
+ return nil, err
+ }
+ res.Releases = append(res.Releases, releases...)
+
+ return res, nil
+}
+
+// listDashCommits merges together the commits in the provided
+// branches, sorted by commit time (newest first), skipping skip
+// items, and stopping after commitsPerPage items.
+// If len(branches) > 1, then skip must be zero.
+//
+// It returns the commits, whether more would follow on a later page,
+// and the oldest skipped commit, if any.
+func (s apiService) listDashCommits(proj *maintner.GerritProject, branches []string, commitsPerPage, skip int) (commits []*apipb.DashCommit, truncated bool, oldestSkipped time.Time) {
+ mixBranches := len(branches) > 1
+ if mixBranches && skip > 0 {
+ panic("unsupported skip in mixed mode")
+ }
+ // oldestItem is the oldest item on the page. It's used to
+ // stop iteration early on the 2nd and later branches when
+ // len(branches) > 1.
+ var oldestItem time.Time
+ for _, branch := range branches {
+ gh := proj.Ref("refs/heads/" + branch)
+ if gh == "" {
+ continue
+ }
+ skipped := 0
+ var add []*apipb.DashCommit
+ iter := s.gitLogIter(gh)
+ for len(add) < commitsPerPage && iter.HasNext() {
+ c := iter.Take()
+ if c.CommitTime.Before(oldestItem) {
+ break
+ }
+ if skipped >= skip {
+ dc := dashCommit(c)
+ dc.Branch = branch
+ add = append(add, dc)
+ } else {
+ skipped++
+ oldestSkipped = c.CommitTime
+ }
+ }
+ commits = append(commits, add...)
+ if !mixBranches {
+ truncated = iter.HasNext()
+ break
+ }
+
+ sort.Slice(commits, func(i, j int) bool {
+ return commits[i].CommitTimeSec > commits[j].CommitTimeSec
+ })
+ if len(commits) > commitsPerPage {
+ commits = commits[:commitsPerPage]
+ truncated = true
+ }
+ if len(commits) > 0 {
+ oldestItem = time.Unix(commits[len(commits)-1].CommitTimeSec, 0)
+ }
+ }
+ return commits, truncated, oldestSkipped
+}
+
+// addGoCommits populates each commit's GoCommitAtTime and
+// GoCommitLatest values. for the oldest and newest corresponding "go"
+// repo commits, respectively. That way there's at least one
+// associated Go commit (even if empty) on the dashboard when viewing
+// https://build.golang.org/?repo=golang.org/x/net.
+//
+// The provided commits must be from most recent to oldest. The
+// oldestSkipped should be the oldest commit time that's on the page
+// prior to commits, or the zero value for the first (newest) page.
+//
+// The maintner corpus must be read-locked.
+func (s apiService) addGoCommits(oldestSkipped time.Time, commits []*apipb.DashCommit) {
+ if len(commits) == 0 {
+ return
+ }
+ goProj := s.c.Gerrit().Project("go.googlesource.com", "go")
+ if goProj == nil {
+ // Shouldn't happen, except in tests with
+ // an empty maintner corpus.
+ return
+ }
+ // Find the oldest (last) commit.
+ oldestX := time.Unix(commits[len(commits)-1].CommitTimeSec, 0)
+
+ // Collect enough goCommits going back far enough such that we have one that's older
+ // than the oldest repo item on the page.
+ var goCommits []*maintner.GitCommit // newest to oldest
+ lastGoHash := func() string {
+ if len(goCommits) == 0 {
+ return ""
+ }
+ return goCommits[len(goCommits)-1].Hash.String()
+ }
+
+ goIter := s.gitLogIter(goProj.Ref("refs/heads/master"))
+ for goIter.HasNext() {
+ c := goIter.Take()
+ goCommits = append(goCommits, c)
+ if c.CommitTime.Before(oldestX) {
+ break
+ }
+ }
+
+ for i := len(commits) - 1; i >= 0; i-- { // walk from oldest to newest
+ dc := commits[i]
+ var maxGoAge time.Time
+ if i == 0 {
+ maxGoAge = oldestSkipped
+ } else {
+ maxGoAge = time.Unix(commits[i-1].CommitTimeSec, 0)
+ }
+ dc.GoCommitAtTime = lastGoHash()
+ for len(goCommits) >= 2 && goCommits[len(goCommits)-2].CommitTime.Before(maxGoAge) {
+ goCommits = goCommits[:len(goCommits)-1]
+ }
+ dc.GoCommitLatest = lastGoHash()
+ }
+}
+
+// dashRepoHeads returns the DashRepoHead for each Gerrit project on
+// the go.googlesource.com server.
+func (s apiService) dashRepoHeads() (heads []*apipb.DashRepoHead) {
+ s.c.Gerrit().ForeachProjectUnsorted(func(gp *maintner.GerritProject) error {
+ if gp.Server() != "go.googlesource.com" {
+ return nil
+ }
+ gh := gp.Ref("refs/heads/master")
+ if gh == "" {
+ return nil
+ }
+ c := gp.GitCommit(gh.String())
+ if c == nil {
+ return nil
+ }
+ heads = append(heads, &apipb.DashRepoHead{
+ GerritProject: gp.Project(),
+ Commit: dashCommit(c),
+ })
+ return nil
+ })
+ sort.Slice(heads, func(i, j int) bool {
+ return heads[i].GerritProject < heads[j].GerritProject
+ })
+ return
+}
+
+// gitLogIter is a git log iterator.
+type gitLogIter struct {
+ corpus *maintner.Corpus
+ nexth maintner.GitHash
+ nextc *maintner.GitCommit // lazily looked up
+}
+
+// HasNext reports whether there's another commit to be seen.
+func (i *gitLogIter) HasNext() bool {
+ if i.nextc == nil {
+ if i.nexth == "" {
+ return false
+ }
+ i.nextc = i.corpus.GitCommit(i.nexth.String())
+ }
+ return i.nextc != nil
+}
+
+// Take returns the next commit (or nil if none remains) and advances past it.
+func (i *gitLogIter) Take() *maintner.GitCommit {
+ if !i.HasNext() {
+ return nil
+ }
+ ret := i.nextc
+ i.nextc = nil
+ if len(ret.Parents) == 0 {
+ i.nexth = ""
+ } else {
+ // TODO: care about returning the history from both
+ // sides of merge commits? Go has a linear history for
+ // the most part so punting for now. I think the old
+ // build.golang.org datastore model got confused by
+ // this too. In any case, this is like:
+ // git log --first-parent.
+ i.nexth = ret.Parents[0].Hash
+ }
+ return ret
+}
+
+// Peek returns the next commit (or nil if none remains) without advancing past it.
+// The next call to Peek or Take will return it again.
+func (i *gitLogIter) Peek() *maintner.GitCommit {
+ if i.HasNext() {
+ // HasNext guarantees that it populates i.nextc.
+ return i.nextc
+ }
+ return nil
+}
+
+func (s apiService) gitLogIter(start maintner.GitHash) *gitLogIter {
+ return &gitLogIter{
+ corpus: s.c,
+ nexth: start,
+ }
+}
+
+func dashCommit(c *maintner.GitCommit) *apipb.DashCommit {
+ return &apipb.DashCommit{
+ Commit: c.Hash.String(),
+ CommitTimeSec: c.CommitTime.Unix(),
+ AuthorName: c.Author.Name(),
+ AuthorEmail: c.Author.Email(),
+ Title: c.Summary(),
+ }
+}
+
+// dashRepoToGerritProj maps a DashboardRequest.repo value to
+// a go.googlesource.com Gerrit project name.
+func dashRepoToGerritProj(repo string) (proj string, err error) {
+ if repo == "go" || repo == "" {
+ return "go", nil
+ }
+ ri, ok := repos.ByImportPath[repo]
+ if !ok || ri.GoGerritProject == "" {
+ return "", grpc.Errorf(codes.NotFound, `unknown repo %q; must be empty, "go", or "golang.org/*"`, repo)
+ }
+ return ri.GoGerritProject, nil
+}
diff --git a/maintner/maintnerd/maintapi/api_test.go b/maintner/maintnerd/maintapi/api_test.go
index 78b7206..88b53b9 100644
--- a/maintner/maintnerd/maintapi/api_test.go
+++ b/maintner/maintnerd/maintapi/api_test.go
@@ -9,6 +9,7 @@
"encoding/hex"
"flag"
"fmt"
+ "strings"
"sync"
"testing"
"time"
@@ -18,6 +19,8 @@
"golang.org/x/build/maintner"
"golang.org/x/build/maintner/godata"
"golang.org/x/build/maintner/maintnerd/apipb"
+ "grpc.go4.org"
+ "grpc.go4.org/codes"
)
func TestGetRef(t *testing.T) {
@@ -318,6 +321,206 @@
}
}
+func TestGetDashboard(t *testing.T) {
+ c := getGoData(t)
+ s := apiService{c}
+
+ type check func(t *testing.T, res *apipb.DashboardResponse, resErr error)
+ var noError check = func(t *testing.T, res *apipb.DashboardResponse, resErr error) {
+ t.Helper()
+ if resErr != nil {
+ t.Fatalf("GetDashboard: %v", resErr)
+ }
+ }
+ var commitsTruncated check = func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ t.Helper()
+ if !res.CommitsTruncated {
+ t.Errorf("CommitsTruncated = false; want true")
+ }
+ if len(res.Commits) == 0 {
+ t.Errorf("no commits; expected some commits when expecting CommitsTruncated")
+ }
+
+ }
+ hasBranch := func(branch string) check {
+ return func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ ok := false
+ for _, b := range res.Branches {
+ if b == branch {
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ t.Errorf("didn't find expected branch %q; got branches: %q", branch, res.Branches)
+ }
+ }
+ }
+ hasRepoHead := func(proj string) check {
+ return func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ ok := false
+ var got []string
+ for _, rh := range res.RepoHeads {
+ if rh.GerritProject == proj {
+ ok = true
+ }
+ got = append(got, rh.GerritProject)
+ }
+ if !ok {
+ t.Errorf("didn't find expected repo head %q; got: %q", proj, got)
+ }
+ }
+ }
+ var hasThreeReleases check = func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ t.Helper()
+ var got []string
+ var gotMaster int
+ var gotReleaseBranch int
+ var uniq = map[string]bool{}
+ for _, r := range res.Releases {
+ got = append(got, r.BranchName)
+ uniq[r.BranchName] = true
+ if r.BranchName == "master" {
+ gotMaster++
+ }
+ if strings.HasPrefix(r.BranchName, "release-branch.go") {
+ gotReleaseBranch++
+ }
+ }
+ if len(uniq) != 3 {
+ t.Errorf("expected 3 Go releases, got: %q", got)
+ }
+ if gotMaster != 1 {
+ t.Errorf("expected 1 Go release to be master, got: %q", got)
+ }
+ if gotReleaseBranch != 2 {
+ t.Errorf("expected 2 Go releases to be release branches, got: %q", got)
+ }
+ }
+ wantRPCError := func(code codes.Code) check {
+ return func(t *testing.T, _ *apipb.DashboardResponse, err error) {
+ if grpc.Code(err) != code {
+ t.Errorf("expected RPC code %v; got %v (err %v)", code, grpc.Code(err), err)
+ }
+ }
+ }
+ basicChecks := []check{
+ noError,
+ commitsTruncated,
+ hasBranch("master"),
+ hasBranch("release-branch.go1.4"),
+ hasBranch("release-branch.go1.13"),
+ hasRepoHead("net"),
+ hasRepoHead("sys"),
+ hasThreeReleases,
+ }
+
+ tests := []struct {
+ name string
+ req *apipb.DashboardRequest
+ checks []check
+ }{
+ // Verify that the default view (with no options) works.
+ {
+ name: "zero_value",
+ req: &apipb.DashboardRequest{},
+ checks: basicChecks,
+ },
+ // Or with explicit values:
+ {
+ name: "zero_value_effectively",
+ req: &apipb.DashboardRequest{
+ Repo: "go",
+ Branch: "master",
+ },
+ checks: basicChecks,
+ },
+ // Max commits:
+ {
+ name: "max_commits",
+ req: &apipb.DashboardRequest{MaxCommits: 1},
+ checks: []check{
+ noError,
+ commitsTruncated,
+ func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ if got, want := len(res.Commits), 1; got != want {
+ t.Errorf("got %v commits; want %v", got, want)
+ }
+ },
+ },
+ },
+ // Verify that branch=mixed doesn't return an error at least.
+ {
+ name: "mixed",
+ req: &apipb.DashboardRequest{Branch: "mixed"},
+ checks: []check{
+ noError,
+ commitsTruncated,
+ hasRepoHead("sys"),
+ hasThreeReleases,
+ },
+ },
+ // Verify non-Go repos:
+ {
+ name: "non_go_repo",
+ req: &apipb.DashboardRequest{Repo: "golang.org/x/net"},
+ checks: []check{
+ noError,
+ commitsTruncated,
+ func(t *testing.T, res *apipb.DashboardResponse, _ error) {
+ for _, c := range res.Commits {
+ if c.GoCommitAtTime == "" {
+ t.Errorf("response contains commit without GoCommitAtTime")
+ }
+ if c.GoCommitLatest == "" {
+ t.Errorf("response contains commit without GoCommitLatest")
+ }
+ if t.Failed() {
+ return
+ }
+ }
+ },
+ },
+ },
+
+ // Validate rejection of bad requests:
+ {
+ name: "bad-repo",
+ req: &apipb.DashboardRequest{Repo: "NOT_EXIST"},
+ checks: []check{wantRPCError(codes.NotFound)},
+ },
+ {
+ name: "bad-branch",
+ req: &apipb.DashboardRequest{Branch: "NOT_EXIST"},
+ checks: []check{wantRPCError(codes.NotFound)},
+ },
+ {
+ name: "mixed-with-pagination",
+ req: &apipb.DashboardRequest{Branch: "mixed", Page: 5},
+ checks: []check{wantRPCError(codes.InvalidArgument)},
+ },
+ {
+ name: "negative-page",
+ req: &apipb.DashboardRequest{Page: -1},
+ checks: []check{wantRPCError(codes.InvalidArgument)},
+ },
+ {
+ name: "too-big-page",
+ req: &apipb.DashboardRequest{Page: 1e6},
+ checks: []check{wantRPCError(codes.InvalidArgument)},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ res, err := s.GetDashboard(context.Background(), tt.req)
+ for _, c := range tt.checks {
+ c(t, res, err)
+ }
+ })
+ }
+}
+
type gerritProject struct {
refs []refHash
}
diff --git a/maintner/maintq/maintq.go b/maintner/maintq/maintq.go
index 7cf7a3a..f600fbc 100644
--- a/maintner/maintq/maintq.go
+++ b/maintner/maintq/maintq.go
@@ -14,9 +14,11 @@
"fmt"
"log"
"net/http"
+ "os"
"sort"
"strings"
+ "github.com/golang/protobuf/proto"
"golang.org/x/build/maintner/maintnerd/apipb"
"golang.org/x/net/http2"
"grpc.go4.org"
@@ -54,6 +56,7 @@
"get-ref": callGetRef,
"try-work": callTryWork,
"list-releases": callListReleases,
+ "get-dashboard": callGetDashboard,
}
log.SetFlags(0)
if flag.NArg() == 0 || cmdFunc[flag.Arg(0)] == nil {
@@ -112,8 +115,7 @@
if err != nil {
return err
}
- fmt.Println(res)
- return nil
+ return printTextProto(res)
}
func callListReleases(args []string) error {
@@ -124,8 +126,33 @@
if err != nil {
return err
}
- for _, r := range res.Releases {
- fmt.Println(r)
+ return printTextProto(res)
+}
+
+func callGetDashboard(args []string) error {
+ req := &apipb.DashboardRequest{}
+
+ fs := flag.NewFlagSet("get-dash-commits", flag.ExitOnError)
+ var page int
+ fs.IntVar(&page, "page", 0, "0-based page number")
+ fs.StringVar(&req.Branch, "branch", "", "branch name; empty means master")
+ fs.StringVar(&req.Repo, "repo", "", "repo name; empty means the main repo, otherwise \"golang.org/*\"")
+ fs.Parse(args)
+ if fs.NArg() != 0 {
+ fs.Usage()
+ os.Exit(2)
}
- return nil
+
+ req.Page = int32(page)
+
+ res, err := mc.GetDashboard(ctx, req)
+ if err != nil {
+ return err
+ }
+ return printTextProto(res)
+}
+
+func printTextProto(m proto.Message) error {
+ tm := proto.TextMarshaler{Compact: false}
+ return tm.Marshal(os.Stdout, m)
}
diff --git a/repos/repos.go b/repos/repos.go
new file mode 100644
index 0000000..4c810d2
--- /dev/null
+++ b/repos/repos.go
@@ -0,0 +1,109 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package repos contains information about Go source repositories.
+package repos
+
+import "fmt"
+
+type Repo struct {
+ // GoGerritProject, if non-empty, is its Gerrit project name,
+ // such as "go", "net", or "sys".
+ GoGerritProject string
+
+ // ImportPath is the repo's import path.
+ // It is empty for the main Go repo.
+ ImportPath string
+
+ MirroredToGithub bool
+
+ // HideFromDashboard, if true, makes the repo not appear at build.golang.org.
+ HideFromDashboard bool
+
+ // CoordinatorCanBuild reports whether this a repo that the
+ // build coordinator knows how to build.
+ CoordinatorCanBuild bool
+}
+
+// ByGerritProject maps from a Gerrit project name ("go", "net", etc)
+// to the Repo's information.
+var ByGerritProject = map[string]*Repo{ /* initialized below */ }
+
+// ByImportPath maps from an import path ("golang.org/x/net") to the
+// Repo's information.
+var ByImportPath = map[string]*Repo{ /* initialized below */ }
+
+func init() {
+ add(&Repo{GoGerritProject: "go", MirroredToGithub: true, CoordinatorCanBuild: true})
+ add(&Repo{GoGerritProject: "dl", MirroredToGithub: true, ImportPath: "golang.org/dl", HideFromDashboard: true, CoordinatorCanBuild: true})
+ add(&Repo{GoGerritProject: "protobuf", MirroredToGithub: true, ImportPath: "github.com/google/protobuf", HideFromDashboard: true})
+ add(&Repo{GoGerritProject: "gddo", MirroredToGithub: true, ImportPath: "github.com/golang/gddo", HideFromDashboard: true})
+ add(&Repo{GoGerritProject: "gofrontend", MirroredToGithub: true, HideFromDashboard: true})
+ add(&Repo{GoGerritProject: "gollvm", MirroredToGithub: true, HideFromDashboard: true})
+ add(&Repo{GoGerritProject: "grpc-review", MirroredToGithub: false, HideFromDashboard: true})
+ x("arch")
+ x("benchmarks")
+ x("blog")
+ x("build")
+ x("crypto")
+ x("debug")
+ x("example", noDash)
+ x("exp")
+ x("image")
+ x("lint", noDash)
+ x("mobile")
+ x("mod")
+ x("net")
+ x("oauth2")
+ x("perf")
+ x("playground", noDash)
+ x("review")
+ x("scratch", noDash)
+ x("sync")
+ x("sys")
+ x("talks")
+ x("term")
+ x("text")
+ x("time")
+ x("tools")
+ x("tour", noDash)
+ x("vgo", noDash)
+ x("website")
+ x("xerrors", noDash)
+}
+
+type modifyRepo func(*Repo)
+
+// noDash is an option to the x func that marks the repo as hidden on
+// the https://build.golang.org/ dashboard.
+func noDash(r *Repo) { r.HideFromDashboard = true }
+
+// x adds a golang.org/x repo.
+func x(proj string, opts ...modifyRepo) {
+ repo := &Repo{
+ GoGerritProject: proj,
+ MirroredToGithub: true,
+ CoordinatorCanBuild: true,
+ ImportPath: "golang.org/x/" + proj,
+ }
+ for _, o := range opts {
+ o(repo)
+ }
+ add(repo)
+}
+
+func add(r *Repo) {
+ if p := r.GoGerritProject; p != "" {
+ if _, dup := ByGerritProject[p]; dup {
+ panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
+ }
+ ByGerritProject[p] = r
+ }
+ if p := r.ImportPath; p != "" {
+ if _, dup := ByImportPath[p]; dup {
+ panic(fmt.Sprintf("duplicate import path %q in %+v", p, r))
+ }
+ ByImportPath[p] = r
+ }
+}
diff --git a/types/types.go b/types/types.go
index a068ae6..6e81729 100644
--- a/types/types.go
+++ b/types/types.go
@@ -31,10 +31,6 @@
// Revision is the full git hash of the repo.
Revision string `json:"revision"`
- // ParentRevisions is the full git hashes of the parents of
- // Revision.
- ParentRevisions []string `json:"parentRevisions"`
-
// GoRevision is the full git hash of the "go" repo, if Repo is not "go" itself.
// Otherwise this is empty.
GoRevision string `json:"goRevision,omitempty"`
