app/appengine/handler.go

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/build/app/cache"
	"golang.org/x/build/app/key"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

const (
	commitsPerPage = 30
	watcherVersion = 3 // must match dashboard/watcher/watcher.go
	builderVersion = 1 // must match dashboard/builder/http.go
)

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		err := datastore.Get(c, com.Key(c), com)
		if com.Num == 0 && com.Desc == "" {
			// Perf builder might have written an incomplete Commit.
			// Pretend it doesn't exist, so that we can get complete details.
			err = datastore.ErrNoSuchEntity
		}
		if err != nil {
			if err == datastore.ErrNoSuchEntity {
				// This error string is special.
				// The commit watcher expects it.
				// Do not change it.
				return nil, errors.New("Commit not found")
			}
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		if com.Num == 0 {
			// Corrupt state which shouldn't happen but does.
			// Return an error so builders' commit loops will
			// be willing to retry submitting this commit.
			return nil, errors.New("in datastore with zero Num")
		}
		if com.Desc == "" || com.User == "" {
			// Also shouldn't happen, but at least happened
			// once on a single commit when trying to fix data
			// in the datastore viewer UI?
			return nil, errors.New("missing field")
		}
		// Strip potentially large and unnecessary fields.
		com.ResultData = nil
		com.PerfResults = nil
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	if !isMasterKey(c, r.FormValue("key")) {
		return nil, errors.New("can only POST commits with master key")
	}

	// For now, the commit watcher doesn't support gccgo.
	// TODO(adg,cmang): remove this exception when gccgo is supported.
	if dashboardForRequest(r) != gccgoDash {
		v, _ := strconv.Atoi(r.FormValue("version"))
		if v != watcherVersion {
			return nil, fmt.Errorf("rejecting POST from commit watcher; need version %v instead of %v",
				watcherVersion, v)
		}
	}

	// POST request
	body, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		return nil, fmt.Errorf("reading Body: %v", err)
	}
	if !bytes.Contains(body, needsBenchmarkingBytes) {
		log.Warningf(c, "old builder detected at %v", r.RemoteAddr)
		return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
	}
	if err := json.Unmarshal(body, com); err != nil {
		return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
	}
	com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c context.Context) error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

var needsBenchmarkingBytes = []byte(`"NeedsBenchmarking"`)

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c context.Context, com *Commit) error {
	var ec Commit // existing commit
	isUpdate := false
	err := datastore.Get(c, com.Key(c), &ec)
	if err != nil && err != datastore.ErrNoSuchEntity {
		return fmt.Errorf("getting Commit: %v", err)
	}
	if err == nil {
		// Commit already in the datastore. Any fields different?
		// If not, don't do anything.
		changes := (com.Num != 0 && com.Num != ec.Num) ||
			com.ParentHash != ec.ParentHash ||
			com.Desc != ec.Desc ||
			com.User != ec.User ||
			!com.Time.Equal(ec.Time)
		if !changes {
			return nil
		}
		ec.ParentHash = com.ParentHash
		ec.Desc = com.Desc
		ec.User = com.User
		if !com.Time.IsZero() {
			ec.Time = com.Time
		}
		if com.Num != 0 {
			ec.Num = com.Num
		}
		isUpdate = true
		com = &ec
	}
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	if com.Num == 0 {
		// get the next commit number
		com.Num = p.NextNum
		p.NextNum++
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			return fmt.Errorf("putting Package: %v", err)
		}
	} else if com.Num >= p.NextNum {
		p.NextNum = com.Num + 1
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			return fmt.Errorf("putting Package: %v", err)
		}
	}
	// if this isn't the first Commit test the parent commit exists.
	// The all zeros are returned by hg's p1node template for parentless commits.
	if com.ParentHash != "" && com.ParentHash != "0000000000000000000000000000000000000000" && com.ParentHash != "0000" {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return errors.New("parent commit not found")
		}
	} else if com.Num != 1 {
		// This is the first commit; fail if it is not number 1.
		// (This will happen if we try to upload a new/different repo
		// where there is already commit data. A bad thing to do.)
		return errors.New("this package already has a first commit; aborting")
	}
	// Update the relevant Tag entity, if applicable.
	if !isUpdate && p.Path == "" {
		var t *Tag
		if com.Branch == "master" {
			t = &Tag{Kind: "tip", Hash: com.Hash}
		}
		if strings.HasPrefix(com.Branch, "release-branch.") {
			t = &Tag{Kind: "release", Name: com.Branch, Hash: com.Hash}
		}
		if t != nil {
			if _, err = datastore.Put(c, t.Key(c), t); err != nil {
				return fmt.Errorf("putting Tag: %v", err)
			}
		}
	}
	// put the Commit
	if err = putCommit(c, com); err != nil {
		return err
	}
	if com.NeedsBenchmarking {
		// add to CommitRun
		cr, err := GetCommitRun(c, com.Num)
		if err != nil {
			return err
		}
		if err = cr.AddCommit(c, com); err != nil {
			return err
		}
		// create PerfResult
		res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
			return fmt.Errorf("putting PerfResult: %v", err)
		}
		// Update perf todo if necessary.
		if err = AddCommitToPerfTodo(c, com); err != nil {
			return err
		}
	}
	return nil
}

// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
// request body and updates the Tag entity for the Kind of tag provided.
//
// This handler is used by a gobuilder process in -commit mode.
func tagHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	t := new(Tag)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
		return nil, err
	}
	if err := t.Valid(); err != nil {
		return nil, err
	}
	c := contextForRequest(r)
	defer cache.Tick(c)
	_, err := datastore.Put(c, t.Key(c), t)
	return nil, err
}

// Todo is a todoHandler response.
type Todo struct {
	Kind string // "build-go-commit" or "build-package"
	Data interface{}
}

// todoHandler returns the next action to be performed by a builder.
// It expects "builder" and "kind" query parameters and returns a *Todo value.
// Multiple "kind" parameters may be specified.
func todoHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	now := cache.Now(c)
	key := "build-todo-" + r.Form.Encode()
	var todo *Todo
	if cache.Get(c, r, now, key, &todo) {
		// Hack to avoid storing nil in memcache.
		if todo.Kind == "none" {
			return nil, nil
		}
		return todo, nil
	}
	var err error
	builder := r.FormValue("builder")
	if builderKeyRevoked(builder) {
		return nil, fmt.Errorf("builder key revoked; no work given")
	}
	for _, kind := range r.Form["kind"] {
		var com *Commit
		switch kind {
		case "build-go-commit":
			com, err = buildTodo(c, builder, "", "")
			if com != nil {
				com.PerfResults = []string{}
			}
		case "build-package":
			packagePath := r.FormValue("packagePath")
			goHash := r.FormValue("goHash")
			com, err = buildTodo(c, builder, packagePath, goHash)
			if com != nil {
				com.PerfResults = []string{}
			}
		case "benchmark-go-commit":
			com, err = perfTodo(c, builder)
		}
		if com != nil || err != nil {
			if com != nil {
				// ResultData can be large and not needed on builder.
				com.ResultData = []string{}
			}
			todo = &Todo{Kind: kind, Data: com}
			break
		}
	}
	if err == nil {
		// Hack to avoid storing nil in memcache.
		if todo == nil {
			todo = &Todo{Kind: "none"}
		}
		cache.Set(c, r, now, key, todo)
	}
	// Hack to avoid storing nil in memcache.
	if todo.Kind == "none" {
		return nil, nil
	}
	return todo, err
}

// buildTodo returns the next Commit to be built (or nil if none available).
//
// If packagePath and goHash are empty, it scans the first 20 Go Commits in
// Num-descending order and returns the first one it finds that doesn't have a
// Result for this builder.
//
// If provided with non-empty packagePath and goHash args, it scans the first
// 20 Commits in Num-descending order for the specified packagePath and
// returns the first that doesn't have a Result for this builder and goHash.
func buildTodo(c context.Context, builder, packagePath, goHash string) (*Commit, error) {
	p, err := GetPackage(c, packagePath)
	if err != nil {
		return nil, err
	}

	t := datastore.NewQuery("Commit").
		Ancestor(p.Key(c)).
		Limit(commitsPerPage).
		Order("-Num").
		Run(c)
	for {
		com := new(Commit)
		if _, err := t.Next(com); err == datastore.Done {
			break
		} else if err != nil {
			return nil, err
		}
		if com.Result(builder, goHash) == nil {
			return com, nil
		}
	}

	// Nothing left to do if this is a package (not the Go tree).
	if packagePath != "" {
		return nil, nil
	}

	// If there are no Go tree commits left to build,
	// see if there are any subrepo commits that need to be built at tip.
	// If so, ask the builder to build a go tree at the tip commit.
	// TODO(adg): do the same for "weekly" and "release" tags.

	tag, err := GetTag(c, "tip", "")
	if err != nil {
		return nil, err
	}

	// Check that this Go commit builds OK for this builder.
	// If not, don't re-build as the subrepos will never get built anyway.
	com, err := tag.Commit(c)
	if err != nil {
		return nil, err
	}
	if r := com.Result(builder, ""); r != nil && !r.OK {
		return nil, nil
	}

	pkgs, err := Packages(c, "subrepo")
	if err != nil {
		return nil, err
	}
	for _, pkg := range pkgs {
		com, err := pkg.LastCommit(c)
		if err != nil {
			log.Warningf(c, "%v: no Commit found: %v", pkg, err)
			continue
		}
		if com.Result(builder, tag.Hash) == nil {
			return tag.Commit(c)
		}
	}

	return nil, nil
}

// perfTodo returns the next Commit to be benchmarked (or nil if none available).
func perfTodo(c context.Context, builder string) (*Commit, error) {
	p := &Package{}
	todo := &PerfTodo{Builder: builder}
	err := datastore.Get(c, todo.Key(c), todo)
	if err != nil && err != datastore.ErrNoSuchEntity {
		return nil, fmt.Errorf("fetching PerfTodo: %v", err)
	}
	if err == datastore.ErrNoSuchEntity {
		todo, err = buildPerfTodo(c, builder)
		if err != nil {
			return nil, err
		}
	}
	if len(todo.CommitNums) == 0 {
		return nil, nil
	}

	// Have commit to benchmark, fetch it.
	num := todo.CommitNums[len(todo.CommitNums)-1]
	t := datastore.NewQuery("Commit").
		Ancestor(p.Key(c)).
		Filter("Num =", num).
		Limit(1).
		Run(c)
	com := new(Commit)
	if _, err := t.Next(com); err != nil {
		return nil, err
	}
	if !com.NeedsBenchmarking {
		return nil, fmt.Errorf("commit from perf todo queue is not intended for benchmarking")
	}

	// Remove benchmarks from other builders.
	var benchs []string
	for _, b := range com.PerfResults {
		bb := strings.Split(b, "|")
		if bb[0] == builder && bb[1] != "meta-done" {
			benchs = append(benchs, bb[1])
		}
	}
	com.PerfResults = benchs

	return com, nil
}

// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction.
func buildPerfTodo(c context.Context, builder string) (*PerfTodo, error) {
	todo := &PerfTodo{Builder: builder}
	tx := func(c context.Context) error {
		err := datastore.Get(c, todo.Key(c), todo)
		if err != nil && err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("fetching PerfTodo: %v", err)
		}
		if err == nil {
			return nil
		}
		t := datastore.NewQuery("CommitRun").
			Ancestor((&Package{}).Key(c)).
			Order("-StartCommitNum").
			Run(c)
		var nums []int
		var releaseNums []int
	loop:
		for {
			cr := new(CommitRun)
			if _, err := t.Next(cr); err == datastore.Done {
				break
			} else if err != nil {
				return fmt.Errorf("scanning commit runs for perf todo: %v", err)
			}
			for i := len(cr.Hash) - 1; i >= 0; i-- {
				if !cr.NeedsBenchmarking[i] || cr.Hash[i] == "" {
					continue // There's nothing to see here. Move along.
				}
				num := cr.StartCommitNum + i
				for k, v := range knownTags {
					// Releases are benchmarked first, because they are important (and there are few of them).
					if cr.Hash[i] == v {
						releaseNums = append(releaseNums, num)
						if k == "go1" {
							break loop // Point of no benchmark: test/bench/shootout: update timing.log to Go 1.
						}
					}
				}
				nums = append(nums, num)
			}
		}
		todo.CommitNums = orderPerfTodo(nums)
		todo.CommitNums = append(todo.CommitNums, releaseNums...)
		if _, err = datastore.Put(c, todo.Key(c), todo); err != nil {
			return fmt.Errorf("putting PerfTodo: %v", err)
		}
		return nil
	}
	return todo, datastore.RunInTransaction(c, tx, nil)
}

func removeCommitFromPerfTodo(c context.Context, builder string, num int) error {
	todo := &PerfTodo{Builder: builder}
	err := datastore.Get(c, todo.Key(c), todo)
	if err != nil && err != datastore.ErrNoSuchEntity {
		return fmt.Errorf("fetching PerfTodo: %v", err)
	}
	if err == datastore.ErrNoSuchEntity {
		return nil
	}
	for i := len(todo.CommitNums) - 1; i >= 0; i-- {
		if todo.CommitNums[i] == num {
			for ; i < len(todo.CommitNums)-1; i++ {
				todo.CommitNums[i] = todo.CommitNums[i+1]
			}
			todo.CommitNums = todo.CommitNums[:i]
			_, err = datastore.Put(c, todo.Key(c), todo)
			if err != nil {
				return fmt.Errorf("putting PerfTodo: %v", err)
			}
			break
		}
	}
	return nil
}

// packagesHandler returns a list of the non-Go Packages monitored
// by the dashboard.
func packagesHandler(r *http.Request) (interface{}, error) {
	kind := r.FormValue("kind")
	c := contextForRequest(r)
	now := cache.Now(c)
	key := "build-packages-" + kind
	var p []*Package
	if cache.Get(c, r, now, key, &p) {
		return p, nil
	}
	p, err := Packages(c, kind)
	if err != nil {
		return nil, err
	}
	cache.Set(c, r, now, key, p)
	return p, nil
}

// buildingHandler records that a build is in progress.
// The data is only stored in memcache and with a timeout. It's assumed
// that the build system will periodically refresh this if the build
// is slow.
func buildingHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	c := contextForRequest(r)
	key := buildingKey(r.FormValue("hash"), r.FormValue("gohash"), r.FormValue("builder"))
	err := memcache.Set(c, &memcache.Item{
		Key:        key,
		Value:      []byte(r.FormValue("url")),
		Expiration: 15 * time.Minute,
	})
	if err != nil {
		return nil, err
	}
	return map[string]interface{}{
		"key": key,
	}, nil
}

// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	// For now, the gccgo builders are using the old stuff.
	// TODO(adg,cmang): remove this exception when gccgo is updated.
	if dashboardForRequest(r) != gccgoDash {
		v, _ := strconv.Atoi(r.FormValue("version"))
		if v != builderVersion {
			return nil, fmt.Errorf("rejecting POST from builder; need version %v instead of %v",
				builderVersion, v)
		}
	}

	c := contextForRequest(r)
	res := new(Result)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if err := res.Valid(); err != nil {
		return nil, fmt.Errorf("validating Result: %v", err)
	}
	defer cache.Tick(c)
	// store the Log text if supplied
	if len(res.Log) > 0 {
		hash, err := PutLog(c, res.Log)
		if err != nil {
			return nil, fmt.Errorf("putting Log: %v", err)
		}
		res.LogHash = hash
	}
	tx := func(c context.Context) error {
		// check Package exists
		if _, err := GetPackage(c, res.PackagePath); err != nil {
			return fmt.Errorf("GetPackage: %v", err)
		}
		// put Result
		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
			return fmt.Errorf("putting Result: %v", err)
		}
		// add Result to Commit
		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
		if err := com.AddResult(c, res); err != nil {
			return fmt.Errorf("AddResult: %v", err)
		}
		// Send build failure notifications, if necessary.
		// Note this must run after the call AddResult, which
		// populates the Commit's ResultData field.
		return notifyOnFailure(c, com, res.Builder)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// perf-result request payload
type PerfRequest struct {
	Builder   string
	Benchmark string
	Hash      string
	OK        bool
	Metrics   []PerfMetric
	Artifacts []PerfArtifact
}

type PerfMetric struct {
	Type string
	Val  uint64
}

type PerfArtifact struct {
	Type string
	Body string
}

// perfResultHandler records a becnhmarking result.
func perfResultHandler(r *http.Request) (interface{}, error) {
	defer r.Body.Close()
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	req := new(PerfRequest)
	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}

	c := contextForRequest(r)
	defer cache.Tick(c)

	// store the text files if supplied
	for i, a := range req.Artifacts {
		hash, err := PutLog(c, a.Body)
		if err != nil {
			return nil, fmt.Errorf("putting Log: %v", err)
		}
		req.Artifacts[i].Body = hash
	}
	tx := func(c context.Context) error {
		return addPerfResult(c, r, req)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// addPerfResult creates PerfResult and updates Commit, PerfTodo,
// PerfMetricRun and PerfConfig.
// MUST be called from inside a transaction.
func addPerfResult(c context.Context, r *http.Request, req *PerfRequest) error {
	// check Package exists
	p, err := GetPackage(c, "")
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	// add result to Commit
	com := &Commit{Hash: req.Hash}
	if err := com.AddPerfResult(c, req.Builder, req.Benchmark); err != nil {
		return fmt.Errorf("AddPerfResult: %v", err)
	}

	// add the result to PerfResult
	res := &PerfResult{CommitHash: req.Hash}
	if err := datastore.Get(c, res.Key(c), res); err != nil {
		return fmt.Errorf("getting PerfResult: %v", err)
	}
	present := res.AddResult(req)
	if _, err := datastore.Put(c, res.Key(c), res); err != nil {
		return fmt.Errorf("putting PerfResult: %v", err)
	}

	// Meta-done denotes that there are no benchmarks left.
	if req.Benchmark == "meta-done" {
		// Don't send duplicate emails for the same commit/builder.
		// And don't send emails about too old commits.
		if !present && com.Num >= p.NextNum-commitsPerPage {
			if err := checkPerfChanges(c, r, com, req.Builder, res); err != nil {
				return err
			}
		}
		if err := removeCommitFromPerfTodo(c, req.Builder, com.Num); err != nil {
			return nil
		}
		return nil
	}

	// update PerfConfig
	newBenchmark, err := UpdatePerfConfig(c, r, req)
	if err != nil {
		return fmt.Errorf("updating PerfConfig: %v", err)
	}
	if newBenchmark {
		// If this is a new benchmark on the builder, delete PerfTodo.
		// It will be recreated later with all commits again.
		todo := &PerfTodo{Builder: req.Builder}
		err = datastore.Delete(c, todo.Key(c))
		if err != nil && err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("deleting PerfTodo: %v", err)
		}
	}

	// add perf metrics
	for _, metric := range req.Metrics {
		m, err := GetPerfMetricRun(c, req.Builder, req.Benchmark, metric.Type, com.Num)
		if err != nil {
			return fmt.Errorf("GetPerfMetrics: %v", err)
		}
		if err = m.AddMetric(c, com.Num, metric.Val); err != nil {
			return fmt.Errorf("AddMetric: %v", err)
		}
	}

	return nil
}

// MUST be called from inside a transaction.
func checkPerfChanges(c context.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
	pc, err := GetPerfConfig(c, r)
	if err != nil {
		return err
	}

	results := res.ParseData()[builder]
	rcNewer := MakePerfResultCache(c, com, true)
	rcOlder := MakePerfResultCache(c, com, false)

	// Check whether we need to send failure notification email.
	if results["meta-done"].OK {
		// This one is successful, see if the next is failed.
		nextRes, err := rcNewer.Next(com.Num)
		if err != nil {
			return err
		}
		if nextRes != nil && isPerfFailed(nextRes, builder) {
			sendPerfFailMail(c, builder, nextRes)
		}
	} else {
		// This one is failed, see if the previous is successful.
		prevRes, err := rcOlder.Next(com.Num)
		if err != nil {
			return err
		}
		if prevRes != nil && !isPerfFailed(prevRes, builder) {
			sendPerfFailMail(c, builder, res)
		}
	}

	// Now see if there are any performance changes.
	// Find the previous and the next results for performance comparison.
	prevRes, err := rcOlder.NextForComparison(com.Num, builder)
	if err != nil {
		return err
	}
	nextRes, err := rcNewer.NextForComparison(com.Num, builder)
	if err != nil {
		return err
	}
	if results["meta-done"].OK {
		// This one is successful, compare with a previous one.
		if prevRes != nil {
			if err := comparePerfResults(c, pc, builder, prevRes, res); err != nil {
				return err
			}
		}
		// Compare a next one with the current.
		if nextRes != nil {
			if err := comparePerfResults(c, pc, builder, res, nextRes); err != nil {
				return err
			}
		}
	} else {
		// This one is failed, compare a previous one with a next one.
		if prevRes != nil && nextRes != nil {
			if err := comparePerfResults(c, pc, builder, prevRes, nextRes); err != nil {
				return err
			}
		}
	}

	return nil
}

func comparePerfResults(c context.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
	changes := significantPerfChanges(pc, builder, prevRes, res)
	if len(changes) == 0 {
		return nil
	}
	com := &Commit{Hash: res.CommitHash}
	if err := datastore.Get(c, com.Key(c), com); err != nil {
		return fmt.Errorf("getting commit %v: %v", com.Hash, err)
	}
	sendPerfMailLater.Call(c, com, prevRes.CommitHash, builder, changes) // add task to queue
	return nil
}

// logHandler displays log text for a given hash.
// It handles paths like "/log/hash".
func logHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "text/plain; charset=utf-8")
	c := contextForRequest(r)
	hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
	key := datastore.NewKey(c, "Log", hash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, key, l); err != nil {
		if err == datastore.ErrNoSuchEntity {
			// Fall back to default namespace;
			// maybe this was on the old dashboard.
			c := appengine.NewContext(r)
			key := datastore.NewKey(c, "Log", hash, 0, nil)
			err = datastore.Get(c, key, l)
		}
		if err != nil {
			logErr(w, r, err)
			return
		}
	}
	b, err := l.Text()
	if err != nil {
		logErr(w, r, err)
		return
	}
	w.Write(b)
}

// clearResultsHandler purges the last commitsPerPage results for the given builder.
// It optionally takes a comma-separated list of specific hashes to clear.
func clearResultsHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	builder := r.FormValue("builder")
	if builder == "" {
		return nil, errors.New("must specify a builder")
	}
	clearAll := r.FormValue("hash") == ""
	hash := strings.Split(r.FormValue("hash"), ",")

	c := contextForRequest(r)
	defer cache.Tick(c)
	pkg := (&Package{}).Key(c) // TODO(adg): support clearing sub-repos
	err := datastore.RunInTransaction(c, func(c context.Context) error {
		var coms []*Commit
		keys, err := datastore.NewQuery("Commit").
			Ancestor(pkg).
			Order("-Num").
			Limit(commitsPerPage).
			GetAll(c, &coms)
		if err != nil {
			return err
		}
		var rKeys []*datastore.Key
		for _, com := range coms {
			if !(clearAll || contains(hash, com.Hash)) {
				continue
			}
			r := com.Result(builder, "")
			if r == nil {
				continue
			}
			com.RemoveResult(r)
			rKeys = append(rKeys, r.Key(c))
		}
		_, err = datastore.PutMulti(c, keys, coms)
		if err != nil {
			return err
		}
		return datastore.DeleteMulti(c, rKeys)
	}, nil)
	return nil, err
}

type dashHandler func(*http.Request) (interface{}, error)

type dashResponse struct {
	Response interface{}
	Error    string
}

// errBadMethod is returned by a dashHandler when
// the request has an unsuitable method.
type errBadMethod string

func (e errBadMethod) Error() string {
	return "bad method: " + string(e)
}

func builderKeyRevoked(builder string) bool {
	switch builder {
	case "plan9-amd64-mischief":
		// Broken and unmaintained for months.
		// It's polluting the dashboard.
		return true
	case "linux-arm-onlinenet":
		// Requested to be revoked by Dave Cheney.
		// The machine is in a fail+report loop
		// and can't be accessed. Revoke it for now.
		return true
	}
	return false
}

// AuthHandler wraps a http.HandlerFunc with a handler that validates the
// supplied key and builder query parameters.
func AuthHandler(h dashHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		c := contextForRequest(r)

		// Put the URL Query values into r.Form to avoid parsing the
		// request body when calling r.FormValue.
		r.Form = r.URL.Query()

		var err error
		var resp interface{}

		// Validate key query parameter for POST requests only.
		key := r.FormValue("key")
		builder := r.FormValue("builder")
		if r.Method == "POST" && !validKey(c, key, builder) {
			err = fmt.Errorf("invalid key %q for builder %q", key, builder)
		}

		// Call the original HandlerFunc and return the response.
		if err == nil {
			resp, err = h(r)
		}

		// Write JSON response.
		dashResp := &dashResponse{Response: resp}
		if err != nil {
			log.Errorf(c, "%v", err)
			dashResp.Error = err.Error()
		}
		w.Header().Set("Content-Type", "application/json")
		if err = json.NewEncoder(w).Encode(dashResp); err != nil {
			log.Criticalf(c, "encoding response: %v", err)
		}
	}
}

func keyHandler(w http.ResponseWriter, r *http.Request) {
	builder := r.FormValue("builder")
	if builder == "" {
		logErr(w, r, errors.New("must supply builder in query string"))
		return
	}
	c := contextForRequest(r)
	fmt.Fprint(w, builderKey(c, builder))
}

func init() {
	// admin handlers
	handleFunc("/init", initHandler)
	handleFunc("/key", keyHandler)

	// authenticated handlers
	handleFunc("/building", AuthHandler(buildingHandler))
	handleFunc("/clear-results", AuthHandler(clearResultsHandler))
	handleFunc("/commit", AuthHandler(commitHandler))
	handleFunc("/packages", AuthHandler(packagesHandler))
	handleFunc("/perf-result", AuthHandler(perfResultHandler))
	handleFunc("/result", AuthHandler(resultHandler))
	handleFunc("/tag", AuthHandler(tagHandler))
	handleFunc("/todo", AuthHandler(todoHandler))

	// public handlers
	handleFunc("/log/", logHandler)
}

func validHash(hash string) bool {
	// TODO(adg): correctly validate a hash
	return hash != ""
}

func validKey(c context.Context, key, builder string) bool {
	if isMasterKey(c, key) {
		return true
	}
	if builderKeyRevoked(builder) {
		return false
	}
	return key == builderKey(c, builder)
}

func isMasterKey(c context.Context, k string) bool {
	return appengine.IsDevAppServer() || k == key.Secret(c)
}

func builderKey(c context.Context, builder string) string {
	h := hmac.New(md5.New, []byte(key.Secret(c)))
	h.Write([]byte(builder))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func logErr(w http.ResponseWriter, r *http.Request, err error) {
	c := contextForRequest(r)
	log.Errorf(c, "Error: %v", err)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprint(w, "Error: ", html.EscapeString(err.Error()))
}

func contextForRequest(r *http.Request) context.Context {
	return dashboardForRequest(r).Context(appengine.NewContext(r))
}

// limitStringLength essentially does return s[:max],
// but it ensures that we dot not split UTF-8 rune in half.
// Otherwise appengine python scripts will break badly.
func limitStringLength(s string, max int) string {
	if len(s) <= max {
		return s
	}
	for {
		s = s[:max]
		r, size := utf8.DecodeLastRuneInString(s)
		if r != utf8.RuneError || size != 1 {
			return s
		}
		max--
	}
}