internal/buildgo/buildgo.go - build - Git at Google

 // Copyright 2017 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // Package buildgo provides tools for pushing and building the Go
 // distribution on buildlets.
 package buildgo

 import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"path"
 	"time"

 	"golang.org/x/build/buildenv"
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/dashboard"
 	"golang.org/x/build/internal/sourcecache"
 	"golang.org/x/build/internal/spanlog"
 )

 // BuilderRev is a build configuration type and a revision.
 type BuilderRev struct {
 	Name string // e.g. "linux-amd64-race"
 	Rev  string // lowercase hex core repo git hash

 	// optional sub-repository details (both must be present)
 	SubName string // e.g. "net"
 	SubRev  string // lowercase hex sub-repo git hash
 }

 func (br BuilderRev) IsSubrepo() bool {
 	return br.SubName != ""
 }

 func (br BuilderRev) SubRevOrGoRev() string {
 	if br.SubRev != "" {
 		return br.SubRev
 	}
 	return br.Rev
 }

 func (br BuilderRev) RepoOrGo() string {
 	if br.SubName == "" {
 		return "go"
 	}
 	return br.SubName
 }

 // SnapshotObjectName is the cloud storage object name of the
 // built Go tree for this builder and Go rev (not the sub-repo).
 // The entries inside this tarball do not begin with "go/".
 func (br *BuilderRev) SnapshotObjectName() string {
 	return fmt.Sprintf("%v/%v/%v.tar.gz", "go", br.Name, br.Rev)
 }

 // SnapshotURL is the absolute URL of the snapshot object (see above).
 func (br *BuilderRev) SnapshotURL(buildEnv *buildenv.Environment) string {
 	return buildEnv.SnapshotURL(br.Name, br.Rev)
 }

 var TestHookSnapshotExists func(*BuilderRev) bool

 // snapshotExists reports whether the snapshot exists in storage.
 // It returns potentially false negatives on network errors.
 // Callers must not depend on this as more than an optimization.
 func (br *BuilderRev) SnapshotExists(ctx context.Context, buildEnv *buildenv.Environment) bool {
 	if f := TestHookSnapshotExists; f != nil {
 		return f(br)
 	}
 	req, err := http.NewRequest("HEAD", br.SnapshotURL(buildEnv), nil)
 	if err != nil {
 		panic(err)
 	}
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 	res, err := http.DefaultClient.Do(req.WithContext(ctx))
 	if err != nil {
 		log.Printf("SnapshotExists check: %v", err)
 		return false
 	}
 	return res.StatusCode == http.StatusOK
 }

 // A GoBuilder knows how to build a revision of Go with the given configuration.
 type GoBuilder struct {
 	spanlog.Logger
 	BuilderRev
 	Conf *dashboard.BuildConfig
 	// Goroot is a Unix-style path relative to the work directory of the builder (e.g. "go").
 	Goroot string
 }

 // RunMake builds the tool chain.
 // goroot is relative to the workdir with forward slashes.
 // w is the Writer to send build output to.
 // remoteErr and err are as described at the top of this file.
 func (gb GoBuilder) RunMake(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
 	// Build the source code.
 	makeSpan := gb.CreateSpan("make", gb.Conf.MakeScript())
 	remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, gb.Conf.MakeScript()), buildlet.ExecOpts{
 		Output:   w,
 		ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
 		Debug:    true,
 		Args:     gb.Conf.MakeScriptArgs(),
 	})
 	if err != nil {
 		makeSpan.Done(err)
 		return nil, err
 	}
 	if remoteErr != nil {
 		makeSpan.Done(remoteErr)
 		return fmt.Errorf("make script failed: %v", remoteErr), nil
 	}
 	makeSpan.Done(nil)

 	// Need to run "go install -race std" before the snapshot + tests.
 	if pkgs := gb.Conf.GoInstallRacePackages(); len(pkgs) > 0 {
 		sp := gb.CreateSpan("install_race_std")
 		remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
 			Output:   w,
 			ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
 			Debug:    true,
 			Args:     append([]string{"install", "-race"}, pkgs...),
 		})
 		if err != nil {
 			sp.Done(err)
 			return nil, err
 		}
 		if remoteErr != nil {
 			sp.Done(err)
 			return fmt.Errorf("go install -race std failed: %v", remoteErr), nil
 		}
 		sp.Done(nil)
 	}

 	if gb.Name == "linux-amd64-racecompile" {
 		return gb.runConcurrentGoBuildStdCmd(ctx, bc, w)
 	}

 	return nil, nil
 }

 // runConcurrentGoBuildStdCmd is a step specific only to the
 // "linux-amd64-racecompile" builder to exercise the Go 1.9's new
 // concurrent compilation. It re-builds the standard library and tools
 // with -gcflags=-c=8 using a race-enabled cmd/compile and cmd/link
 // (built by caller, RunMake, per builder config).
 // The idea is that this might find data races in cmd/compile and cmd/link.
 func (gb GoBuilder) runConcurrentGoBuildStdCmd(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
 	span := gb.CreateSpan("go_build_c128_std_cmd")
 	remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
 		Output:   w,
 		ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
 		Debug:    true,
 		Args:     []string{"build", "-a", "-gcflags=-c=8", "std", "cmd"},
 	})
 	if err != nil {
 		span.Done(err)
 		return nil, err
 	}
 	if remoteErr != nil {
 		span.Done(remoteErr)
 		return fmt.Errorf("go build failed: %v", remoteErr), nil
 	}
 	span.Done(nil)
 	return nil, nil
 }

 // FetchSubrepo checks out the go.googlesource.com repository
 // repo (for example, "net" or "oauth2") at git revision rev,
 // and places it into the buildlet's GOPATH workspace at
 // $GOPATH/src/<importPath>.
 //
 // The GOPATH workspace is assumed to be the "gopath" directory
 // in the buildlet's work directory.
 func FetchSubrepo(ctx context.Context, sl spanlog.Logger, bc *buildlet.Client, repo, rev, importPath string) error {
 	tgz, err := sourcecache.GetSourceTgz(sl, repo, rev)
 	if err != nil {
 		return err
 	}
 	return bc.PutTar(ctx, tgz, "gopath/src/"+importPath)
 }

 // VersionTgz returns an io.Reader of a *.tar.gz file containing only
 // a VERSION file containing the contents of the provided rev string.
 func VersionTgz(rev string) io.Reader {
 	var buf bytes.Buffer
 	zw := gzip.NewWriter(&buf)
 	tw := tar.NewWriter(zw)

 	// Writing to a bytes.Buffer should never fail, so check
 	// errors with an explosion:
 	check := func(err error) {
 		if err != nil {
 			panic("previously assumed to never fail: " + err.Error())
 		}
 	}

 	contents := fmt.Sprintf("devel " + rev)
 	check(tw.WriteHeader(&tar.Header{
 		Name: "VERSION",
 		Mode: 0644,
 		Size: int64(len(contents)),
 	}))
 	_, err := io.WriteString(tw, contents)
 	check(err)
 	check(tw.Close())
 	check(zw.Close())
 	return bytes.NewReader(buf.Bytes())
 }
	// Copyright 2017 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// Package buildgo provides tools for pushing and building the Go
	// distribution on buildlets.
	package buildgo

	import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"
	"time"

	"golang.org/x/build/buildenv"
	"golang.org/x/build/buildlet"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/internal/sourcecache"
	"golang.org/x/build/internal/spanlog"
	)

	// BuilderRev is a build configuration type and a revision.
	type BuilderRev struct {
	Name string // e.g. "linux-amd64-race"
	Rev string // lowercase hex core repo git hash

	// optional sub-repository details (both must be present)
	SubName string // e.g. "net"
	SubRev string // lowercase hex sub-repo git hash
	}

	func (br BuilderRev) IsSubrepo() bool {
	return br.SubName != ""
	}

	func (br BuilderRev) SubRevOrGoRev() string {
	if br.SubRev != "" {
	return br.SubRev
	}
	return br.Rev
	}

	func (br BuilderRev) RepoOrGo() string {
	if br.SubName == "" {
	return "go"
	}
	return br.SubName
	}

	// SnapshotObjectName is the cloud storage object name of the
	// built Go tree for this builder and Go rev (not the sub-repo).
	// The entries inside this tarball do not begin with "go/".
	func (br *BuilderRev) SnapshotObjectName() string {
	return fmt.Sprintf("%v/%v/%v.tar.gz", "go", br.Name, br.Rev)
	}

	// SnapshotURL is the absolute URL of the snapshot object (see above).
	func (br BuilderRev) SnapshotURL(buildEnv buildenv.Environment) string {
	return buildEnv.SnapshotURL(br.Name, br.Rev)
	}

	var TestHookSnapshotExists func(*BuilderRev) bool

	// snapshotExists reports whether the snapshot exists in storage.
	// It returns potentially false negatives on network errors.
	// Callers must not depend on this as more than an optimization.
	func (br BuilderRev) SnapshotExists(ctx context.Context, buildEnv buildenv.Environment) bool {
	if f := TestHookSnapshotExists; f != nil {
	return f(br)
	}
	req, err := http.NewRequest("HEAD", br.SnapshotURL(buildEnv), nil)
	if err != nil {
	panic(err)
	}
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	res, err := http.DefaultClient.Do(req.WithContext(ctx))
	if err != nil {
	log.Printf("SnapshotExists check: %v", err)
	return false
	}
	return res.StatusCode == http.StatusOK
	}

	// A GoBuilder knows how to build a revision of Go with the given configuration.
	type GoBuilder struct {
	spanlog.Logger
	BuilderRev
	Conf *dashboard.BuildConfig
	// Goroot is a Unix-style path relative to the work directory of the builder (e.g. "go").
	Goroot string
	}

	// RunMake builds the tool chain.
	// goroot is relative to the workdir with forward slashes.
	// w is the Writer to send build output to.
	// remoteErr and err are as described at the top of this file.
	func (gb GoBuilder) RunMake(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
	// Build the source code.
	makeSpan := gb.CreateSpan("make", gb.Conf.MakeScript())
	remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, gb.Conf.MakeScript()), buildlet.ExecOpts{
	Output: w,
	ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
	Debug: true,
	Args: gb.Conf.MakeScriptArgs(),
	})
	if err != nil {
	makeSpan.Done(err)
	return nil, err
	}
	if remoteErr != nil {
	makeSpan.Done(remoteErr)
	return fmt.Errorf("make script failed: %v", remoteErr), nil
	}
	makeSpan.Done(nil)

	// Need to run "go install -race std" before the snapshot + tests.
	if pkgs := gb.Conf.GoInstallRacePackages(); len(pkgs) > 0 {
	sp := gb.CreateSpan("install_race_std")
	remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
	Output: w,
	ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
	Debug: true,
	Args: append([]string{"install", "-race"}, pkgs...),
	})
	if err != nil {
	sp.Done(err)
	return nil, err
	}
	if remoteErr != nil {
	sp.Done(err)
	return fmt.Errorf("go install -race std failed: %v", remoteErr), nil
	}
	sp.Done(nil)
	}

	if gb.Name == "linux-amd64-racecompile" {
	return gb.runConcurrentGoBuildStdCmd(ctx, bc, w)
	}

	return nil, nil
	}

	// runConcurrentGoBuildStdCmd is a step specific only to the
	// "linux-amd64-racecompile" builder to exercise the Go 1.9's new
	// concurrent compilation. It re-builds the standard library and tools
	// with -gcflags=-c=8 using a race-enabled cmd/compile and cmd/link
	// (built by caller, RunMake, per builder config).
	// The idea is that this might find data races in cmd/compile and cmd/link.
	func (gb GoBuilder) runConcurrentGoBuildStdCmd(ctx context.Context, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
	span := gb.CreateSpan("go_build_c128_std_cmd")
	remoteErr, err = bc.Exec(ctx, path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
	Output: w,
	ExtraEnv: append(gb.Conf.Env(), "GOBIN="),
	Debug: true,
	Args: []string{"build", "-a", "-gcflags=-c=8", "std", "cmd"},
	})
	if err != nil {
	span.Done(err)
	return nil, err
	}
	if remoteErr != nil {
	span.Done(remoteErr)
	return fmt.Errorf("go build failed: %v", remoteErr), nil
	}
	span.Done(nil)
	return nil, nil
	}

	// FetchSubrepo checks out the go.googlesource.com repository
	// repo (for example, "net" or "oauth2") at git revision rev,
	// and places it into the buildlet's GOPATH workspace at
	// $GOPATH/src/<importPath>.
	//
	// The GOPATH workspace is assumed to be the "gopath" directory
	// in the buildlet's work directory.
	func FetchSubrepo(ctx context.Context, sl spanlog.Logger, bc *buildlet.Client, repo, rev, importPath string) error {
	tgz, err := sourcecache.GetSourceTgz(sl, repo, rev)
	if err != nil {
	return err
	}
	return bc.PutTar(ctx, tgz, "gopath/src/"+importPath)
	}

	// VersionTgz returns an io.Reader of a *.tar.gz file containing only
	// a VERSION file containing the contents of the provided rev string.
	func VersionTgz(rev string) io.Reader {
	var buf bytes.Buffer
	zw := gzip.NewWriter(&buf)
	tw := tar.NewWriter(zw)

	// Writing to a bytes.Buffer should never fail, so check
	// errors with an explosion:
	check := func(err error) {
	if err != nil {
	panic("previously assumed to never fail: " + err.Error())
	}
	}

	contents := fmt.Sprintf("devel " + rev)
	check(tw.WriteHeader(&tar.Header{
	Name: "VERSION",
	Mode: 0644,
	Size: int64(len(contents)),
	}))
	_, err := io.WriteString(tw, contents)
	check(err)
	check(tw.Close())
	check(zw.Close())
	return bytes.NewReader(buf.Bytes())
	}