blob: 9aa8a503db3db9a3557ba52a3b197c5f976d350d [file] [log] [blame]
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TODO(adg): packages at weekly/release
// TODO(adg): some means to register new packages
// +build appengine
package build
import (
"bytes"
"context"
"encoding/json"
"fmt"
"html/template"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
"golang.org/x/build/app/cache"
"golang.org/x/build/dashboard"
"golang.org/x/build/types"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
)
// isDevAppServer is whether we're running locally with dev_appserver.py.
// This is the documented way to check which environment we're running in, per:
// https://cloud.google.com/appengine/docs/standard/python/tools/using-local-server#detecting_application_runtime_environment
var isDevAppServer = !strings.HasPrefix(os.Getenv("SERVER_SOFTWARE"), "Google App Engine/")
func init() {
handleFunc("/", uiHandler)
}
// uiHandler draws the build status page.
func uiHandler(w http.ResponseWriter, r *http.Request) {
d := goDash
c := d.Context(appengine.NewContext(r))
now := cache.Now(c)
key := "build-ui"
mode := r.FormValue("mode")
page, _ := strconv.Atoi(r.FormValue("page"))
if page < 0 {
page = 0
}
key += fmt.Sprintf("-page%v", page)
repo := r.FormValue("repo")
if repo != "" {
key += "-repo-" + repo
}
branch := r.FormValue("branch")
switch branch {
case "all":
branch = ""
case "":
branch = "master"
}
if repo != "" || mode == "json" {
// Don't filter on branches in sub-repos.
// TODO(adg): figure out how to make this work sensibly.
// Don't filter on branches in json mode.
branch = ""
}
if branch != "" {
key += "-branch-" + branch
}
hashes := r.Form["hash"]
var data uiTemplateData
if len(hashes) > 0 || !cache.Get(c, r, now, key, &data) {
pkg := &Package{} // empty package is the main repository
if repo != "" {
var err error
pkg, err = GetPackage(c, repo)
if err != nil {
logErr(w, r, err)
return
}
}
var commits []*Commit
var err error
if len(hashes) > 0 {
commits, err = fetchCommits(c, pkg, hashes)
} else {
commits, err = dashCommits(c, pkg, page, branch)
}
if err != nil {
logErr(w, r, err)
return
}
builders := commitBuilders(commits)
branches := listBranches(c)
var tagState []*TagState
// Only show sub-repo state on first page of normal repo view.
if pkg.Kind == "" && len(hashes) == 0 && page == 0 && (branch == "" || branch == "master") {
s, err := GetTagState(c, "tip", "")
if err != nil {
if err == datastore.ErrNoSuchEntity {
if isDevAppServer {
goto BuildData
}
err = fmt.Errorf("tip tag not found")
}
logErr(w, r, err)
return
}
tagState = []*TagState{s}
for _, b := range branches {
if !strings.HasPrefix(b, "release-branch.") {
continue
}
if hiddenBranches[b] {
continue
}
s, err := GetTagState(c, "release", b)
if err == datastore.ErrNoSuchEntity {
continue
}
if err != nil {
logErr(w, r, err)
return
}
tagState = append(tagState, s)
}
}
// Sort tagState in reverse lexical order by name so higher
// numbered release branches show first for subrepos
// https://build.golang.org/ after master. We want the subrepo
// order to be "master, release-branch.go1.12,
// release-branch.go1.11" so they're in order by date (newest
// first). If we weren't already at two digit minor versions we'd
// need to parse the branch name, but we can be lazy now
// and just do a string compare.
sort.Slice(tagState, func(i, j int) bool { // is item 'i' less than item 'j'?
ni, nj := tagState[i].Name, tagState[j].Name
switch {
case ni == "master":
return true // an i of "master" is always first
case nj == "master":
return false // if i wasn't "master", it can't be less than j's "master"
default:
return ni > nj // "release-branch.go1.12" > "release-branch.go1.11", so 1.12 sorts earlier
}
})
BuildData:
p := &Pagination{}
if len(commits) == commitsPerPage {
p.Next = page + 1
}
if page > 0 {
p.Prev = page - 1
p.HasPrev = true
}
data = uiTemplateData{
Package: pkg,
Commits: commits,
Builders: builders,
TagState: tagState,
Pagination: p,
Branches: branches,
Branch: branch,
}
if len(hashes) == 0 {
cache.Set(c, r, now, key, &data)
}
}
data.Dashboard = d
switch mode {
case "failures":
failuresHandler(w, r, &data)
return
case "json":
jsonHandler(w, r, &data)
return
}
// Populate building URLs for the HTML UI only.
data.populateBuildingURLs(c)
var buf bytes.Buffer
if err := uiTemplate.Execute(&buf, &data); err != nil {
logErr(w, r, err)
return
}
buf.WriteTo(w)
}
func listBranches(c context.Context) (branches []string) {
var commits []*Commit
_, err := datastore.NewQuery("Commit").Distinct().Project("Branch").GetAll(c, &commits)
if err != nil {
log.Errorf(c, "listBranches: %v", err)
return
}
for _, c := range commits {
if strings.HasPrefix(c.Branch, "release-branch.go") &&
strings.HasSuffix(c.Branch, "-security") {
continue
}
branches = append(branches, c.Branch)
}
return
}
// failuresHandler is https://build.golang.org/?mode=failures , where it outputs
// one line per failure on the front page, in the form:
// hash builder failure-url
func failuresHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
w.Header().Set("Content-Type", "text/plain")
d := goDash
for _, c := range data.Commits {
for _, b := range data.Builders {
res := c.Result(b, "")
if res == nil || res.OK || res.LogHash == "" {
continue
}
url := fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
fmt.Fprintln(w, c.Hash, b, url)
}
}
}
// jsonHandler is https://build.golang.org/?mode=json
// The output is a types.BuildStatus JSON object.
func jsonHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
d := goDash
// cell returns one of "" (no data), "ok", or a failure URL.
cell := func(res *Result) string {
switch {
case res == nil:
return ""
case res.OK:
return "ok"
}
return fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
}
var res types.BuildStatus
res.Builders = data.Builders
// First the commits from the main section (the "go" repo)
for _, c := range data.Commits {
rev := types.BuildRevision{
Repo: "go",
Results: make([]string, len(data.Builders)),
}
commitToBuildRevision(c, &rev)
for i, b := range data.Builders {
rev.Results[i] = cell(c.Result(b, ""))
}
res.Revisions = append(res.Revisions, rev)
}
// Then the one commit each for the subrepos for each of the tracked tags.
// (tip, Go 1.4, etc)
for _, ts := range data.TagState {
for _, pkgState := range ts.Packages {
goRev := ts.Tag.Hash
goBranch := ts.Name
if goBranch == "tip" {
// Normalize old hg terminology into
// our git branch name.
goBranch = "master"
}
rev := types.BuildRevision{
Repo: pkgState.Package.Name,
GoRevision: goRev,
Results: make([]string, len(data.Builders)),
GoBranch: goBranch,
}
commitToBuildRevision(pkgState.Commit, &rev)
for i, b := range res.Builders {
rev.Results[i] = cell(pkgState.Commit.Result(b, goRev))
}
res.Revisions = append(res.Revisions, rev)
}
}
v, _ := json.MarshalIndent(res, "", "\t")
w.Header().Set("Content-Type", "text/json; charset=utf-8")
w.Write(v)
}
// commitToBuildRevision fills in the fields of BuildRevision rev that
// are derived from Commit c.
func commitToBuildRevision(c *Commit, rev *types.BuildRevision) {
rev.Revision = c.Hash
// TODO: A comment may have more than one parent.
rev.ParentRevisions = []string{c.ParentHash}
rev.Date = c.Time.Format(time.RFC3339)
rev.Branch = c.Branch
rev.Author = c.User
rev.Desc = c.Desc
}
type Pagination struct {
Next, Prev int
HasPrev bool
}
// dashCommits gets a slice of the latest Commits to the current dashboard.
// If page > 0 it paginates by commitsPerPage.
func dashCommits(c context.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
offset := page * commitsPerPage
q := datastore.NewQuery("Commit").
Ancestor(pkg.Key(c)).
Order("-Num")
if branch != "" {
q = q.Filter("Branch =", branch)
}
var commits []*Commit
_, err := q.Limit(commitsPerPage).Offset(offset).
GetAll(c, &commits)
// If we're running locally and don't have data, return some test data.
// This lets people hack on the UI without setting up gitmirror & friends.
if len(commits) == 0 && isDevAppServer && err == nil {
commits = []*Commit{
{
Hash: "7d7c6a97f815e9279d08cfaea7d5efb5e90695a8",
ParentHash: "",
Num: 1,
User: "bwk",
Desc: "hello, world",
},
}
}
return commits, err
}
// fetchCommits gets a slice of the specific commit hashes
func fetchCommits(c context.Context, pkg *Package, hashes []string) ([]*Commit, error) {
var out []*Commit
var keys []*datastore.Key
for _, hash := range hashes {
commit := &Commit{
Hash: hash,
PackagePath: pkg.Path,
}
out = append(out, commit)
keys = append(keys, commit.Key(c))
}
err := datastore.GetMulti(c, keys, out)
return out, err
}
// commitBuilders returns the names of the builders that provided
// Results for the provided commits.
func commitBuilders(commits []*Commit) []string {
builders := make(map[string]bool)
for _, commit := range commits {
for _, r := range commit.Results() {
builders[r.Builder] = true
}
}
// Add all known builders from the builder configuration too.
// We want to see columns even if there are no results so we
// can identify missing builders. (Issue 19930)
for name, bc := range dashboard.Builders {
if !bc.BuildsRepoPostSubmit("go", "master", "master") {
continue
}
builders[name] = true
}
k := keys(builders)
sort.Sort(builderOrder(k))
return k
}
func keys(m map[string]bool) (s []string) {
s = make([]string, 0, len(m))
for k := range m {
s = append(s, k)
}
sort.Strings(s)
return
}
// builderOrder implements sort.Interface, sorting builder names
// ("darwin-amd64", etc) first by builderPriority and then alphabetically.
type builderOrder []string
func (s builderOrder) Len() int { return len(s) }
func (s builderOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s builderOrder) Less(i, j int) bool {
pi, pj := builderPriority(s[i]), builderPriority(s[j])
if pi == pj {
return s[i] < s[j]
}
return pi < pj
}
func builderPriority(builder string) (p int) {
// Put -temp builders at the end, always.
if strings.HasSuffix(builder, "-temp") {
defer func() { p += 20 }()
}
// Group race builders together.
if isRace(builder) {
return 2
}
// If the OS has a specified priority, use it.
if p, ok := osPriority[builderOS(builder)]; ok {
return p
}
// The rest.
return 10
}
func isRace(s string) bool {
return strings.Contains(s, "-race-") || strings.HasSuffix(s, "-race")
}
func unsupported(builder string) bool {
if strings.HasSuffix(builder, "-temp") {
return true
}
return unsupportedOS(builderOS(builder))
}
func unsupportedOS(os string) bool {
if os == "race" || os == "android" || os == "all" {
return false
}
p, ok := osPriority[os]
return !ok || p > 1
}
// Priorities for specific operating systems.
var osPriority = map[string]int{
"all": 0,
"darwin": 1,
"freebsd": 1,
"linux": 1,
"windows": 1,
// race == 2
"android": 3,
"openbsd": 4,
"netbsd": 5,
"dragonfly": 6,
}
// TagState represents the state of all Packages at a Tag.
type TagState struct {
Name string // "tip", "release-branch.go1.4", etc
Tag *Commit
Packages []*PackageState
}
// Branch returns the git branch name, converting from the old
// terminology we used from Go's hg days into git terminology.
func (ts *TagState) Branch() string {
if ts.Name == "tip" {
return "master"
}
return ts.Name
}
// PackageState represents the state of a Package at a Tag.
type PackageState struct {
Package *Package
Commit *Commit
}
// GetTagState fetches the results for all Go subrepos at the specified Tag.
// (Kind is "tip" or "release"; name is like "release-branch.go1.4".)
func GetTagState(c context.Context, kind, name string) (*TagState, error) {
tag, err := GetTag(c, kind, name)
if err != nil {
return nil, err
}
pkgs, err := Packages(c, "subrepo")
if err != nil {
return nil, err
}
st := TagState{Name: tag.String()}
for _, pkg := range pkgs {
com, err := pkg.LastCommit(c)
if err != nil {
log.Warningf(c, "%v: no Commit found: %v", pkg, err)
continue
}
st.Packages = append(st.Packages, &PackageState{pkg, com})
}
st.Tag, err = tag.Commit(c)
if err != nil {
return nil, err
}
return &st, nil
}
type uiTemplateData struct {
Dashboard *Dashboard
Package *Package
Commits []*Commit
Builders []string
TagState []*TagState
Pagination *Pagination
Branches []string
Branch string
}
// buildingKey returns a memcache key that points to the log URL
// of an inflight build for the given hash, goHash, and builder.
func buildingKey(hash, goHash, builder string) string {
return fmt.Sprintf("building|%v|%v|%v", hash, goHash, builder)
}
// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
// URLs of builds which are currently in progress.
func (td *uiTemplateData) populateBuildingURLs(ctx context.Context) {
// need are memcache keys: "building|<hash>|<gohash>|<builder>"
// The hash is of the main "go" repo, or the subrepo commit hash.
// The gohash is empty for the main repo, else it's the Go hash.
var need []string
commit := map[string]*Commit{} // commit hash -> Commit
// Gather pending commits for main repo.
for _, b := range td.Builders {
for _, c := range td.Commits {
if c.Result(b, "") == nil {
commit[c.Hash] = c
need = append(need, buildingKey(c.Hash, "", b))
}
}
}
// Gather pending commits for sub-repos.
for _, ts := range td.TagState {
goHash := ts.Tag.Hash
for _, b := range td.Builders {
for _, pkg := range ts.Packages {
c := pkg.Commit
commit[c.Hash] = c
if c.Result(b, goHash) == nil {
need = append(need, buildingKey(c.Hash, goHash, b))
}
}
}
}
if len(need) == 0 {
return
}
m, err := memcache.GetMulti(ctx, need)
if err != nil {
// oh well. this is a cute non-critical feature anyway.
log.Debugf(ctx, "GetMulti of building keys: %v", err)
return
}
for k, it := range m {
f := strings.SplitN(k, "|", 4)
if len(f) != 4 {
continue
}
hash, goHash, builder := f[1], f[2], f[3]
c, ok := commit[hash]
if !ok {
continue
}
m := c.buildingURLs
if m == nil {
m = make(map[builderAndGoHash]string)
c.buildingURLs = m
}
m[builderAndGoHash{builder, goHash}] = string(it.Value)
}
}
var uiTemplate = template.Must(
template.New("ui.html").Funcs(tmplFuncs).ParseFiles("ui.html"),
)
var tmplFuncs = template.FuncMap{
"builderSpans": builderSpans,
"builderSubheading": builderSubheading,
"builderSubheading2": builderSubheading2,
"shortDesc": shortDesc,
"shortHash": shortHash,
"shortUser": shortUser,
"tail": tail,
"unsupported": unsupported,
"isUntested": isUntested,
}
func splitDash(s string) (string, string) {
i := strings.Index(s, "-")
if i >= 0 {
return s[:i], s[i+1:]
}
return s, ""
}
// builderOS returns the os tag for a builder string
func builderOS(s string) string {
os, _ := splitDash(s)
return os
}
// builderOSOrRace returns the builder OS or, if it is a race builder, "race".
func builderOSOrRace(s string) string {
if isRace(s) {
return "race"
}
return builderOS(s)
}
// builderArch returns the arch tag for a builder string
func builderArch(s string) string {
_, arch := splitDash(s)
arch, _ = splitDash(arch) // chop third part
return arch
}
// builderSubheading returns a short arch tag for a builder string
// or, if it is a race builder, the builder OS.
func builderSubheading(s string) string {
if isRace(s) {
return builderOS(s)
}
return builderArch(s)
}
// builderSubheading2 returns any third part of a hyphenated builder name.
// For instance, for "linux-amd64-nocgo", it returns "nocgo".
// For race builders it returns the empty string.
func builderSubheading2(s string) string {
if isRace(s) {
return ""
}
_, secondThird := splitDash(s)
_, third := splitDash(secondThird)
return third
}
type builderSpan struct {
N int
OS string
Unsupported bool
}
// builderSpans creates a list of tags showing
// the builder's operating system names, spanning
// the appropriate number of columns.
func builderSpans(s []string) []builderSpan {
var sp []builderSpan
for len(s) > 0 {
i := 1
os := builderOSOrRace(s[0])
u := unsupportedOS(os) || strings.HasSuffix(s[0], "-temp")
for i < len(s) && builderOSOrRace(s[i]) == os {
i++
}
sp = append(sp, builderSpan{i, os, u})
s = s[i:]
}
return sp
}
// shortDesc returns the first line of a description.
func shortDesc(desc string) string {
if i := strings.Index(desc, "\n"); i != -1 {
desc = desc[:i]
}
return limitStringLength(desc, 100)
}
// shortHash returns a short version of a hash.
func shortHash(hash string) string {
if len(hash) > 7 {
hash = hash[:7]
}
return hash
}
// shortUser returns a shortened version of a user string.
func shortUser(user string) string {
if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j {
user = user[i+1 : j]
}
if i := strings.Index(user, "@"); i >= 0 {
return user[:i]
}
return user
}
// tail returns the trailing n lines of s.
func tail(n int, s string) string {
lines := strings.Split(s, "\n")
if len(lines) < n {
return s
}
return strings.Join(lines[len(lines)-n:], "\n")
}