blob: 3862085f8df5c06afadac3b1f8135164aac11a0b [file] [log] [blame]
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package buildgo provides tools for pushing and building the Go
// distribution on buildlets.
package buildgo
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"log"
"net/http"
"path"
"time"
"golang.org/x/build/buildenv"
"golang.org/x/build/buildlet"
"golang.org/x/build/dashboard"
"golang.org/x/build/internal/sourcecache"
"golang.org/x/build/internal/spanlog"
)
// BuilderRev is a build configuration type and a revision.
type BuilderRev struct {
Name string // e.g. "linux-amd64-race"
Rev string // lowercase hex core repo git hash
// optional sub-repository details (both must be present)
SubName string // e.g. "net"
SubRev string // lowercase hex sub-repo git hash
}
func (br BuilderRev) IsSubrepo() bool {
return br.SubName != ""
}
func (br BuilderRev) SubRevOrGoRev() string {
if br.SubRev != "" {
return br.SubRev
}
return br.Rev
}
func (br BuilderRev) RepoOrGo() string {
if br.SubName == "" {
return "go"
}
return br.SubName
}
// SnapshotObjectName is the cloud storage object name of the
// built Go tree for this builder and Go rev (not the sub-repo).
// The entries inside this tarball do not begin with "go/".
func (br *BuilderRev) SnapshotObjectName() string {
return fmt.Sprintf("%v/%v/%v.tar.gz", "go", br.Name, br.Rev)
}
// SnapshotURL is the absolute URL of the snapshot object (see above).
func (br *BuilderRev) SnapshotURL(buildEnv *buildenv.Environment) string {
return buildEnv.SnapshotURL(br.Name, br.Rev)
}
var TestHookSnapshotExists func(*BuilderRev) bool
// snapshotExists reports whether the snapshot exists in storage.
// It returns potentially false negatives on network errors.
// Callers must not depend on this as more than an optimization.
func (br *BuilderRev) SnapshotExists(ctx context.Context, buildEnv *buildenv.Environment) bool {
if f := TestHookSnapshotExists; f != nil {
return f(br)
}
req, err := http.NewRequest("HEAD", br.SnapshotURL(buildEnv), nil)
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
log.Printf("SnapshotExists check: %v", err)
return false
}
return res.StatusCode == http.StatusOK
}
// A GoBuilder knows how to build a revision of Go with the given configuration.
type GoBuilder struct {
spanlog.Logger
BuilderRev
Conf *dashboard.BuildConfig
// Goroot is a Unix-style path relative to the work directory of the builder (e.g. "go").
Goroot string
}
// RunMake builds the tool chain.
// goroot is relative to the workdir with forward slashes.
// w is the Writer to send build output to.
// remoteErr and err are as described at the top of this file.
func (gb GoBuilder) RunMake(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
// Build the source code.
makeSpan := gb.CreateSpan("make", gb.Conf.MakeScript())
remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, gb.Conf.MakeScript()), buildlet.ExecOpts{
Output: w,
ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
Debug: true,
Args: gb.Conf.MakeScriptArgs(),
})
if err != nil {
makeSpan.Done(err)
return nil, err
}
if remoteErr != nil {
makeSpan.Done(remoteErr)
return fmt.Errorf("make script failed: %v", remoteErr), nil
}
makeSpan.Done(nil)
// Need to run "go install -race std" before the snapshot + tests.
if pkgs := gb.Conf.GoInstallRacePackages(); len(pkgs) > 0 {
sp := gb.CreateSpan("install_race_std")
remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
Output: w,
ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
Debug: true,
Args: append([]string{"install", "-race"}, pkgs...),
})
if err != nil {
sp.Done(err)
return nil, err
}
if remoteErr != nil {
sp.Done(err)
return fmt.Errorf("go install -race std failed: %v", remoteErr), nil
}
sp.Done(nil)
}
if gb.Name == "linux-amd64-racecompile" {
return gb.runConcurrentGoBuildStdCmd(ctx, bc, w)
}
return nil, nil
}
// runConcurrentGoBuildStdCmd is a step specific only to the
// "linux-amd64-racecompile" builder to exercise the Go 1.9's new
// concurrent compilation. It re-builds the standard library and tools
// with -gcflags=-c=8 using a race-enabled cmd/compile and cmd/link
// (built by caller, RunMake, per builder config).
// The idea is that this might find data races in cmd/compile and cmd/link.
func (gb GoBuilder) runConcurrentGoBuildStdCmd(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
span := gb.CreateSpan("go_build_c128_std_cmd")
remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
Output: w,
ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
Debug: true,
Args: []string{"build", "-a", "-gcflags=-c=8", "std", "cmd"},
})
if err != nil {
span.Done(err)
return nil, err
}
if remoteErr != nil {
span.Done(remoteErr)
return fmt.Errorf("go build failed: %v", remoteErr), nil
}
span.Done(nil)
return nil, nil
}
// FetchSubrepo checks out the go.googlesource.com repository
// repo (for example, "net" or "oauth2") at git revision rev,
// and places it into the buildlet's GOPATH workspace at
// $GOPATH/src/<importPath>.
//
// The GOPATH workspace is assumed to be the "gopath" directory
// in the buildlet's work directory.
func FetchSubrepo(ctx context.Context, sl spanlog.Logger, bc *buildlet.Client, repo, rev, importPath string) error {
tgz, err := sourcecache.GetSourceTgz(sl, repo, rev)
if err != nil {
return err
}
return bc.PutTar(ctx, tgz, "gopath/src/"+importPath)
}
// VersionTgz returns an io.Reader of a *.tar.gz file containing only
// a VERSION file containing the contents of the provided rev string.
func VersionTgz(rev string) io.Reader {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
// Writing to a bytes.Buffer should never fail, so check
// errors with an explosion:
check := func(err error) {
if err != nil {
panic("previously assumed to never fail: " + err.Error())
}
}
contents := fmt.Sprintf("devel " + rev)
check(tw.WriteHeader(&tar.Header{
Name: "VERSION",
Mode: 0644,
Size: int64(len(contents)),
}))
_, err := io.WriteString(tw, contents)
check(err)
check(tw.Close())
check(zw.Close())
return bytes.NewReader(buf.Bytes())
}