app/appengine/ui.go

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): packages at weekly/release
// TODO(adg): some means to register new packages

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"golang.org/x/build/app/cache"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/types"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

// uiHandler draws the build status page.
func uiHandler(w http.ResponseWriter, r *http.Request) {
	d := goDash
	c := d.Context(appengine.NewContext(r))
	now := cache.Now(c)
	key := "build-ui"

	mode := r.FormValue("mode")

	page, _ := strconv.Atoi(r.FormValue("page"))
	if page < 0 {
		page = 0
	}
	key += fmt.Sprintf("-page%v", page)

	repo := r.FormValue("repo")
	if repo != "" {
		key += "-repo-" + repo
	}

	branch := r.FormValue("branch")
	switch branch {
	case "all":
		branch = ""
	case "":
		branch = "master"
	}
	if repo != "" || mode == "json" {
		// Don't filter on branches in sub-repos.
		// TODO(adg): figure out how to make this work sensibly.
		// Don't filter on branches in json mode.
		branch = ""
	}
	if branch != "" {
		key += "-branch-" + branch
	}

	hashes := r.Form["hash"]

	var data uiTemplateData
	if len(hashes) > 0 || !cache.Get(c, r, now, key, &data) {

		pkg := &Package{} // empty package is the main repository
		if repo != "" {
			var err error
			pkg, err = GetPackage(c, repo)
			if err != nil {
				logErr(w, r, err)
				return
			}
		}
		var commits []*Commit
		var err error
		if len(hashes) > 0 {
			commits, err = fetchCommits(c, pkg, hashes)
		} else {
			commits, err = dashCommits(c, pkg, page, branch)
		}
		if err != nil {
			logErr(w, r, err)
			return
		}
		branches := listBranches(c)
		releaseBranches := supportedReleaseBranches(branches)
		builders := commitBuilders(commits, releaseBranches)

		var tagState []*TagState
		// Only show sub-repo state on first page of normal repo view.
		if pkg.Kind == "" && len(hashes) == 0 && page == 0 && (branch == "" || branch == "master") {
			s, err := GetTagState(c, "tip", "")
			if err != nil {
				if err == datastore.ErrNoSuchEntity {
					if appengine.IsDevAppServer() {
						goto BuildData
					}
					err = fmt.Errorf("tip tag not found")
				}
				logErr(w, r, err)
				return
			}
			tagState = []*TagState{s}
			for _, b := range releaseBranches {
				s, err := GetTagState(c, "release", b)
				if err == datastore.ErrNoSuchEntity {
					continue
				}
				if err != nil {
					logErr(w, r, err)
					return
				}
				tagState = append(tagState, s)
			}
		}
		// Sort tagState in reverse lexical order by name so higher
		// numbered release branches show first for subrepos
		// https://build.golang.org/ after master. We want the subrepo
		// order to be "master, release-branch.go1.12,
		// release-branch.go1.11" so they're in order by date (newest
		// first). If we weren't already at two digit minor versions we'd
		// need to parse the branch name, but we can be lazy now
		// and just do a string compare.
		sort.Slice(tagState, func(i, j int) bool { // is item 'i' less than item 'j'?
			ni, nj := tagState[i].Name, tagState[j].Name
			switch {
			case ni == "master":
				return true // an i of "master" is always first
			case nj == "master":
				return false // if i wasn't "master", it can't be less than j's "master"
			default:
				return ni > nj // "release-branch.go1.12" > "release-branch.go1.11", so 1.12 sorts earlier
			}
		})

	BuildData:
		p := &Pagination{}
		if len(commits) == commitsPerPage {
			p.Next = page + 1
		}
		if page > 0 {
			p.Prev = page - 1
			p.HasPrev = true
		}

		data = uiTemplateData{
			Package:    pkg,
			Commits:    commits,
			Builders:   builders,
			TagState:   tagState,
			Pagination: p,
			Branches:   branches,
			Branch:     branch,
		}
		if len(hashes) == 0 {
			cache.Set(c, r, now, key, &data)
		}
	}
	data.Dashboard = d

	switch mode {
	case "failures":
		failuresHandler(w, r, &data)
		return
	case "json":
		jsonHandler(w, r, &data)
		return
	}

	// Populate building URLs for the HTML UI only.
	data.populateBuildingURLs(c)

	var buf bytes.Buffer
	if err := uiTemplate.Execute(&buf, &data); err != nil {
		logErr(w, r, err)
		return
	}
	buf.WriteTo(w)
}

func listBranches(c context.Context) (branches []string) {
	var commits []*Commit
	_, err := datastore.NewQuery("Commit").Distinct().Project("Branch").GetAll(c, &commits)
	if err != nil {
		log.Errorf(c, "listBranches: %v", err)
		return
	}
	for _, c := range commits {
		if strings.HasPrefix(c.Branch, "release-branch.go") &&
			strings.HasSuffix(c.Branch, "-security") {
			continue
		}
		branches = append(branches, c.Branch)
	}
	return
}

// failuresHandler is https://build.golang.org/?mode=failures , where it outputs
// one line per failure on the front page, in the form:
//    hash builder failure-url
func failuresHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
	w.Header().Set("Content-Type", "text/plain")
	d := goDash
	for _, c := range data.Commits {
		for _, b := range data.Builders {
			res := c.Result(b, "")
			if res == nil || res.OK || res.LogHash == "" {
				continue
			}
			url := fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
			fmt.Fprintln(w, c.Hash, b, url)
		}
	}
}

// jsonHandler is https://build.golang.org/?mode=json
// The output is a types.BuildStatus JSON object.
func jsonHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
	d := goDash

	// cell returns one of "" (no data), "ok", or a failure URL.
	cell := func(res *Result) string {
		switch {
		case res == nil:
			return ""
		case res.OK:
			return "ok"
		}
		return fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
	}

	var res types.BuildStatus
	res.Builders = data.Builders

	// First the commits from the main section (the "go" repo)
	for _, c := range data.Commits {
		rev := types.BuildRevision{
			Repo:    "go",
			Results: make([]string, len(data.Builders)),
		}
		commitToBuildRevision(c, &rev)
		for i, b := range data.Builders {
			rev.Results[i] = cell(c.Result(b, ""))
		}
		res.Revisions = append(res.Revisions, rev)
	}

	// Then the one commit each for the subrepos for each of the tracked tags.
	// (tip, Go 1.4, etc)
	for _, ts := range data.TagState {
		for _, pkgState := range ts.Packages {
			goRev := ts.Tag.Hash
			goBranch := ts.Name
			if goBranch == "tip" {
				// Normalize old hg terminology into
				// our git branch name.
				goBranch = "master"
			}
			rev := types.BuildRevision{
				Repo:       pkgState.Package.Name,
				GoRevision: goRev,
				Results:    make([]string, len(data.Builders)),
				GoBranch:   goBranch,
			}
			commitToBuildRevision(pkgState.Commit, &rev)
			for i, b := range res.Builders {
				rev.Results[i] = cell(pkgState.Commit.Result(b, goRev))
			}
			res.Revisions = append(res.Revisions, rev)
		}
	}

	v, _ := json.MarshalIndent(res, "", "\t")
	w.Header().Set("Content-Type", "text/json; charset=utf-8")
	w.Write(v)
}

// commitToBuildRevision fills in the fields of BuildRevision rev that
// are derived from Commit c.
func commitToBuildRevision(c *Commit, rev *types.BuildRevision) {
	rev.Revision = c.Hash
	// TODO: A comment may have more than one parent.
	rev.ParentRevisions = []string{c.ParentHash}
	rev.Date = c.Time.Format(time.RFC3339)
	rev.Branch = c.Branch
	rev.Author = c.User
	rev.Desc = c.Desc
}

type Pagination struct {
	Next, Prev int
	HasPrev    bool
}

// dashCommits gets a slice of the latest Commits to the current dashboard.
// If page > 0 it paginates by commitsPerPage.
func dashCommits(c context.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
	offset := page * commitsPerPage
	q := datastore.NewQuery("Commit").
		Ancestor(pkg.Key(c)).
		Order("-Num")

	if branch != "" {
		q = q.Filter("Branch =", branch)
	}

	var commits []*Commit
	_, err := q.Limit(commitsPerPage).Offset(offset).
		GetAll(c, &commits)

	// If we're running locally and don't have data, return some test data.
	// This lets people hack on the UI without setting up gitmirror & friends.
	if len(commits) == 0 && appengine.IsDevAppServer() && err == nil {
		commits = []*Commit{
			{
				Hash:       "7d7c6a97f815e9279d08cfaea7d5efb5e90695a8",
				ParentHash: "",
				Num:        1,
				User:       "bwk",
				Desc:       "hello, world",
			},
		}
	}
	return commits, err
}

// fetchCommits gets a slice of the specific commit hashes
func fetchCommits(c context.Context, pkg *Package, hashes []string) ([]*Commit, error) {
	var out []*Commit
	var keys []*datastore.Key
	for _, hash := range hashes {
		commit := &Commit{
			Hash:        hash,
			PackagePath: pkg.Path,
		}
		out = append(out, commit)
		keys = append(keys, commit.Key(c))
	}
	err := datastore.GetMulti(c, keys, out)
	return out, err
}

// commitBuilders returns the names of active builders that provided
// Results for the provided commits.
func commitBuilders(commits []*Commit, releaseBranches []string) []string {
	builders := make(map[string]bool)
	for _, commit := range commits {
		for _, r := range commit.Results() {
			builders[r.Builder] = true
		}
	}
	// Add all known builders from the builder configuration too.
	// We want to see columns even if there are no results so we
	// can identify missing builders. (Issue 19930)
	for name, bc := range dashboard.Builders {
		if !activePostSubmitBuilder(bc, releaseBranches) {
			continue
		}
		builders[name] = true
	}
	k := keys(builders)
	sort.Sort(builderOrder(k))
	return k
}

// activePostSubmitBuilder reports whether the builder bc
// is considered to be "active", meaning it's configured
// to test the Go repository on master branch or at least
// one of the supported release branches.
func activePostSubmitBuilder(bc *dashboard.BuildConfig, releaseBranches []string) bool {
	if bc.BuildsRepoPostSubmit("go", "master", "master") {
		return true
	}
	for _, rb := range releaseBranches {
		if bc.BuildsRepoPostSubmit("go", rb, rb) {
			return true
		}
	}
	// TODO(golang.org/issue/34744): This doesn't catch x-repo-only builders yet; adjust further as needed.
	return false
}

func keys(m map[string]bool) (s []string) {
	s = make([]string, 0, len(m))
	for k := range m {
		s = append(s, k)
	}
	sort.Strings(s)
	return
}

// builderOrder implements sort.Interface, sorting builder names
// ("darwin-amd64", etc) first by builderPriority and then alphabetically.
type builderOrder []string

func (s builderOrder) Len() int      { return len(s) }
func (s builderOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s builderOrder) Less(i, j int) bool {
	pi, pj := builderPriority(s[i]), builderPriority(s[j])
	if pi == pj {
		return s[i] < s[j]
	}
	return pi < pj
}

func builderPriority(builder string) (p int) {
	// Put -temp builders at the end, always.
	if strings.HasSuffix(builder, "-temp") {
		defer func() { p += 20 }()
	}
	// Group race builders together.
	if isRace(builder) {
		return 2
	}
	// If the OS has a specified priority, use it.
	if p, ok := osPriority[builderOS(builder)]; ok {
		return p
	}
	// The rest.
	return 10
}

func isRace(s string) bool {
	return strings.Contains(s, "-race-") || strings.HasSuffix(s, "-race")
}

func unsupported(builder string) bool {
	if strings.HasSuffix(builder, "-temp") {
		return true
	}
	return unsupportedOS(builderOS(builder))
}

func unsupportedOS(os string) bool {
	if os == "race" || os == "android" || os == "all" {
		return false
	}
	p, ok := osPriority[os]
	return !ok || p > 1
}

// Priorities for specific operating systems.
var osPriority = map[string]int{
	"all":     0,
	"darwin":  1,
	"freebsd": 1,
	"linux":   1,
	"windows": 1,
	// race == 2
	"android":   3,
	"openbsd":   4,
	"netbsd":    5,
	"dragonfly": 6,
}

// TagState represents the state of all Packages at a Tag.
type TagState struct {
	Name     string // "tip", "release-branch.go1.4", etc
	Tag      *Commit
	Packages []*PackageState
}

// Branch returns the git branch name, converting from the old
// terminology we used from Go's hg days into git terminology.
func (ts *TagState) Branch() string {
	if ts.Name == "tip" {
		return "master"
	}
	return ts.Name
}

// PackageState represents the state of a Package at a Tag.
type PackageState struct {
	Package *Package
	Commit  *Commit
}

// GetTagState fetches the results for all Go subrepos at the specified Tag.
// (Kind is "tip" or "release"; name is like "release-branch.go1.4".)
func GetTagState(c context.Context, kind, name string) (*TagState, error) {
	tag, err := GetTag(c, kind, name)
	if err != nil {
		return nil, err
	}
	pkgs, err := Packages(c, "subrepo")
	if err != nil {
		return nil, err
	}
	st := TagState{Name: tag.String()}
	for _, pkg := range pkgs {
		com, err := pkg.LastCommit(c)
		if err != nil {
			log.Warningf(c, "%v: no Commit found: %v", pkg, err)
			continue
		}
		st.Packages = append(st.Packages, &PackageState{pkg, com})
	}
	st.Tag, err = tag.Commit(c)
	if err != nil {
		return nil, err
	}
	return &st, nil
}

type uiTemplateData struct {
	Dashboard  *Dashboard
	Package    *Package
	Commits    []*Commit
	Builders   []string
	TagState   []*TagState
	Pagination *Pagination
	Branches   []string
	Branch     string
}

// buildingKey returns a memcache key that points to the log URL
// of an inflight build for the given hash, goHash, and builder.
func buildingKey(hash, goHash, builder string) string {
	return fmt.Sprintf("building|%v|%v|%v", hash, goHash, builder)
}

// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
// URLs of builds which are currently in progress.
func (td *uiTemplateData) populateBuildingURLs(ctx context.Context) {
	// need are memcache keys: "building|<hash>|<gohash>|<builder>"
	// The hash is of the main "go" repo, or the subrepo commit hash.
	// The gohash is empty for the main repo, else it's the Go hash.
	var need []string

	commit := map[string]*Commit{} // commit hash -> Commit

	// Gather pending commits for main repo.
	for _, b := range td.Builders {
		for _, c := range td.Commits {
			if c.Result(b, "") == nil {
				commit[c.Hash] = c
				need = append(need, buildingKey(c.Hash, "", b))
			}
		}
	}

	// Gather pending commits for sub-repos.
	for _, ts := range td.TagState {
		goHash := ts.Tag.Hash
		for _, b := range td.Builders {
			for _, pkg := range ts.Packages {
				c := pkg.Commit
				commit[c.Hash] = c
				if c.Result(b, goHash) == nil {
					need = append(need, buildingKey(c.Hash, goHash, b))
				}
			}
		}
	}

	if len(need) == 0 {
		return
	}

	m, err := memcache.GetMulti(ctx, need)
	if err != nil {
		// oh well. this is a cute non-critical feature anyway.
		log.Debugf(ctx, "GetMulti of building keys: %v", err)
		return
	}
	for k, it := range m {
		f := strings.SplitN(k, "|", 4)
		if len(f) != 4 {
			continue
		}
		hash, goHash, builder := f[1], f[2], f[3]
		c, ok := commit[hash]
		if !ok {
			continue
		}
		m := c.buildingURLs
		if m == nil {
			m = make(map[builderAndGoHash]string)
			c.buildingURLs = m
		}
		m[builderAndGoHash{builder, goHash}] = string(it.Value)
	}

}

var uiTemplate = template.Must(
	template.New("ui.html").Funcs(tmplFuncs).ParseFiles(templateFile("ui.html")),
)

var tmplFuncs = template.FuncMap{
	"builderSpans":       builderSpans,
	"builderSubheading":  builderSubheading,
	"builderSubheading2": builderSubheading2,
	"shortDesc":          shortDesc,
	"shortHash":          shortHash,
	"shortUser":          shortUser,
	"tail":               tail,
	"unsupported":        unsupported,
	"isUntested":         isUntested,
}

func splitDash(s string) (string, string) {
	i := strings.Index(s, "-")
	if i >= 0 {
		return s[:i], s[i+1:]
	}
	return s, ""
}

// builderOS returns the os tag for a builder string
func builderOS(s string) string {
	os, _ := splitDash(s)
	return os
}

// builderOSOrRace returns the builder OS or, if it is a race builder, "race".
func builderOSOrRace(s string) string {
	if isRace(s) {
		return "race"
	}
	return builderOS(s)
}

// builderArch returns the arch tag for a builder string
func builderArch(s string) string {
	_, arch := splitDash(s)
	arch, _ = splitDash(arch) // chop third part
	return arch
}

// builderSubheading returns a short arch tag for a builder string
// or, if it is a race builder, the builder OS.
func builderSubheading(s string) string {
	if isRace(s) {
		return builderOS(s)
	}
	return builderArch(s)
}

// builderSubheading2 returns any third part of a hyphenated builder name.
// For instance, for "linux-amd64-nocgo", it returns "nocgo".
// For race builders it returns the empty string.
func builderSubheading2(s string) string {
	if isRace(s) {
		return ""
	}
	_, secondThird := splitDash(s)
	_, third := splitDash(secondThird)
	return third
}

type builderSpan struct {
	N           int
	OS          string
	Unsupported bool
}

// builderSpans creates a list of tags showing
// the builder's operating system names, spanning
// the appropriate number of columns.
func builderSpans(s []string) []builderSpan {
	var sp []builderSpan
	for len(s) > 0 {
		i := 1
		os := builderOSOrRace(s[0])
		u := unsupportedOS(os) || strings.HasSuffix(s[0], "-temp")
		for i < len(s) && builderOSOrRace(s[i]) == os {
			i++
		}
		sp = append(sp, builderSpan{i, os, u})
		s = s[i:]
	}
	return sp
}

// shortDesc returns the first line of a description.
func shortDesc(desc string) string {
	if i := strings.Index(desc, "\n"); i != -1 {
		desc = desc[:i]
	}
	return limitStringLength(desc, 100)
}

// shortHash returns a short version of a hash.
func shortHash(hash string) string {
	if len(hash) > 7 {
		hash = hash[:7]
	}
	return hash
}

// shortUser returns a shortened version of a user string.
func shortUser(user string) string {
	if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j {
		user = user[i+1 : j]
	}
	if i := strings.Index(user, "@"); i >= 0 {
		return user[:i]
	}
	return user
}

// tail returns the trailing n lines of s.
func tail(n int, s string) string {
	lines := strings.Split(s, "\n")
	if len(lines) < n {
		return s
	}
	return strings.Join(lines[len(lines)-n:], "\n")
}

// templateFile returns the path to the provided HTML template file,
// conditionally prepending a relative path depending on the
// environment.
func templateFile(base string) string {
	// In tests the current directory is ".", but in prod it's up
	// two levels. So just look to see if it's in . first.
	if _, err := os.Stat(base); err == nil {
		return base
	}
	return filepath.Join("app/appengine", base)
}