blob: 400555d3c4125b6fa6c3c4d508cc0e87462a4481 [file] [log] [blame]
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Command release builds a Go release.
package main
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"context"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
"golang.org/x/build"
"golang.org/x/build/buildenv"
"golang.org/x/build/buildlet"
"golang.org/x/build/dashboard"
)
//go:generate go run makestatic.go
var (
target = flag.String("target", "", "If specified, build specific target platform (e.g. 'linux-amd64'). Default is to build all.")
watch = flag.Bool("watch", false, "Watch the build. Only compatible with -target")
stagingDir = flag.String("staging_dir", "", "If specified, use this as the staging directory for untested release artifacts. Default is the system temporary directory.")
rev = flag.String("rev", "", "Go revision to build, alternative to -tarball")
tarball = flag.String("tarball", "", "Go tree tarball to build, alternative to -rev")
version = flag.String("version", "", "Version string (go1.5.2)")
user = flag.String("user", username(), "coordinator username, appended to 'user-'")
skipTests = flag.Bool("skip_tests", false, "skip tests; run make.bash but not all.bash (only use if sufficient testing was done elsewhere)")
uploadMode = flag.Bool("upload", false, "Upload files (exclusive to all other flags)")
uploadKick = flag.String("edge_kick_command", "", "Command to run to kick off an edge cache update")
)
var (
coordClient *buildlet.CoordinatorClient
buildEnv *buildenv.Environment
)
func main() {
flag.Parse()
rand.Seed(time.Now().UnixNano())
if *uploadMode {
buildenv.CheckUserCredentials()
userToken() // Call userToken for the side-effect of exiting if a gomote token doesn't exist.
if err := upload(flag.Args()); err != nil {
log.Fatal(err)
}
return
}
if (*rev == "" && *tarball == "") || (*rev != "" && *tarball != "") {
log.Fatal("must specify one of -rev or -tarball")
}
if *version == "" {
log.Fatal(`must specify -version flag (such as "go1.12" or "go1.13beta1")`)
}
coordClient = coordinatorClient()
buildEnv = buildenv.Production
var wg sync.WaitGroup
matches := 0
for _, b := range builds {
b := b
if *target != "" && b.String() != *target {
continue
}
if !match(b.GoQuery, *version) {
continue
}
matches++
b.logf("Start.")
wg.Add(1)
go func() {
defer wg.Done()
if err := b.make(); err != nil {
b.logf("Error: %v", err)
} else {
b.logf("Done.")
}
}()
}
if *target != "" && matches == 0 {
log.Fatalf("no targets matched %q", *target)
}
wg.Wait()
}
type Build struct {
// GoQuery is a Go version query specifying the Go versions
// the build applies to. Empty string means all Go versions.
GoQuery string
OS, Arch string
Source bool
Race bool // Build race detector.
Builder string // Key for dashboard.Builders.
TestOnly bool // Run tests only; don't produce a release artifact.
Goarm int // GOARM value if set.
SkipTests bool // skip tests (run make.bash but not all.bash); needed by cross-compile builders (s390x)
}
func (b *Build) String() string {
switch {
case b.Source:
return "src"
case b.TestOnly:
// Test-only builds are named after the builder used to
// perform them. For example, "linux-amd64-longtest".
return b.Builder
case b.Goarm != 0:
return fmt.Sprintf("%v-%vv%vl", b.OS, b.Arch, b.Goarm)
default:
return fmt.Sprintf("%v-%v", b.OS, b.Arch)
}
}
func (b *Build) toolDir() string { return "go/pkg/tool/" + b.OS + "_" + b.Arch }
func (b *Build) pkgDir() string { return "go/pkg/" + b.OS + "_" + b.Arch }
func (b *Build) logf(format string, args ...interface{}) {
format = fmt.Sprintf("%v: %s", b, format)
log.Printf(format, args...)
}
var builds = []*Build{
{
Source: true,
Builder: "linux-amd64",
},
{
GoQuery: ">= go1.16beta1",
OS: "linux",
Arch: "386",
Builder: "linux-386-stretch",
},
{
OS: "linux",
Arch: "arm",
Builder: "linux-arm-aws",
Goarm: 6, // For compatibility with all Raspberry Pi models.
},
{
GoQuery: ">= go1.16beta1",
OS: "linux",
Arch: "amd64",
Race: true,
Builder: "linux-amd64-stretch", // Using Stretch as of Go 1.16 because Jessie LTS has ended (golang.org/issue/40561#issuecomment-731482962).
},
{
GoQuery: ">= go1.16beta1",
OS: "linux",
Arch: "arm64",
Builder: "linux-arm64-aws",
},
{
GoQuery: ">= go1.17beta1", // See #45727.
OS: "freebsd",
Arch: "386",
Builder: "freebsd-386-11_4",
},
{
GoQuery: ">= go1.17beta1", // See #45727.
OS: "freebsd",
Arch: "amd64",
Race: true,
Builder: "freebsd-amd64-11_4",
},
{
OS: "windows",
Arch: "386",
Builder: "windows-386-2008",
},
{
OS: "windows",
Arch: "amd64",
Race: true,
Builder: "windows-amd64-2008",
},
{
GoQuery: ">= go1.17beta1", // Go 1.17 Beta 1 is the first Go (pre-)release with the windows/arm64 port.
OS: "windows",
Arch: "arm64",
Race: false, // Not supported as of 2021-06-01.
Builder: "windows-arm64-10",
},
{
GoQuery: ">= go1.17beta1",
OS: "darwin",
Arch: "amd64",
Race: true,
Builder: "darwin-amd64-11_0",
},
{
GoQuery: ">= go1.16beta1", // Go 1.16 Beta 1 is the first Go (pre-)release with the darwin/arm64 port.
OS: "darwin",
Arch: "arm64",
Race: true,
Builder: "darwin-arm64-11_0-toothrot",
},
{
OS: "linux",
Arch: "s390x",
SkipTests: true,
Builder: "linux-s390x-crosscompile",
},
// TODO(bradfitz): switch this ppc64 builder to a Kubernetes
// container cross-compiling ppc64 like the s390x one? For
// now, the ppc64le builders (5) are back, so let's see if we
// can just depend on them not going away.
{
OS: "linux",
Arch: "ppc64le",
SkipTests: true,
Builder: "linux-ppc64le-buildlet",
},
// Older builds.
{
GoQuery: "< go1.16beta1",
OS: "linux",
Arch: "386",
Builder: "linux-386-jessie",
},
{
GoQuery: "< go1.16beta1",
OS: "linux",
Arch: "amd64",
Race: true,
Builder: "linux-amd64-jessie", // Using Jessie for Go 1.11 through Go 1.15 inclusive due to golang.org/issue/31293.
},
{
GoQuery: "< go1.16beta1",
OS: "linux",
Arch: "arm64",
Builder: "linux-arm64-packet",
},
{
GoQuery: "< go1.17beta1", // See #40563.
OS: "freebsd",
Arch: "386",
Builder: "freebsd-386-11_2",
},
{
GoQuery: "< go1.17beta1", // See #40563.
OS: "freebsd",
Arch: "amd64",
Race: true,
Builder: "freebsd-amd64-11_2",
},
{
GoQuery: "< go1.17beta1", // See golang/go#46161.
OS: "darwin",
Arch: "amd64",
Race: true,
Builder: "darwin-amd64-10_15",
},
// Test-only builds.
{
Builder: "linux-386-longtest",
OS: "linux", Arch: "386",
TestOnly: true,
},
{
Builder: "linux-amd64-longtest",
OS: "linux", Arch: "amd64",
TestOnly: true,
},
{
Builder: "windows-amd64-longtest",
OS: "windows", Arch: "amd64",
TestOnly: true,
},
}
var preBuildCleanFiles = []string{
".gitattributes",
".github",
".gitignore",
".hgignore",
".hgtags",
"misc/dashboard",
"misc/makerelease",
}
func (b *Build) buildlet() (*buildlet.Client, error) {
b.logf("Creating buildlet.")
bc, err := coordClient.CreateBuildlet(b.Builder)
if err != nil {
return nil, err
}
bc.SetReleaseMode(true) // disable pargzip; golang.org/issue/19052
return bc, nil
}
func (b *Build) make() error {
ctx := context.TODO()
bc, ok := dashboard.Builders[b.Builder]
if !ok {
return fmt.Errorf("unknown builder: %v", bc)
}
var hostArch string // non-empty if we're cross-compiling (s390x)
if b.SkipTests && bc.IsContainer() && (bc.GOARCH() != "amd64" && bc.GOARCH() != "386") {
hostArch = "amd64"
}
client, err := b.buildlet()
if err != nil {
return err
}
defer client.Close()
work, err := client.WorkDir(ctx)
if err != nil {
return err
}
// Push source to buildlet.
b.logf("Pushing source to buildlet.")
const (
goDir = "go"
goPath = "gopath"
go14 = "go1.4"
)
if *tarball != "" {
tarFile, err := os.Open(*tarball)
if err != nil {
b.logf("failed to open tarball %q: %v", *tarball, err)
return err
}
if err := client.PutTar(ctx, tarFile, goDir); err != nil {
b.logf("failed to put tarball %q into dir %q: %v", *tarball, goDir, err)
return err
}
tarFile.Close()
} else {
tar := "https://go.googlesource.com/go/+archive/" + *rev + ".tar.gz"
if err := client.PutTarFromURL(ctx, tar, goDir); err != nil {
b.logf("failed to put tarball %q into dir %q: %v", tar, goDir, err)
return err
}
}
if u := bc.GoBootstrapURL(buildEnv); u != "" && !b.Source {
b.logf("Installing go1.4.")
if err := client.PutTarFromURL(ctx, u, go14); err != nil {
return err
}
}
// Write out version file.
b.logf("Writing VERSION file.")
if err := client.Put(ctx, strings.NewReader(*version), "go/VERSION", 0644); err != nil {
return err
}
b.logf("Cleaning goroot (pre-build).")
if err := client.RemoveAll(ctx, addPrefix(goDir, preBuildCleanFiles)...); err != nil {
return err
}
if b.Source {
b.logf("Skipping build.")
// Remove unwanted top-level directories and verify only "go" remains:
if err := client.RemoveAll(ctx, "tmp", "gocache"); err != nil {
return err
}
if err := b.checkTopLevelDirs(ctx, client); err != nil {
return fmt.Errorf("verifying no unwanted top-level directories: %v", err)
}
if err := b.checkPerm(ctx, client); err != nil {
return fmt.Errorf("verifying file permissions: %v", err)
}
finalFilename := *version + "." + b.String() + ".tar.gz"
return b.fetchTarball(ctx, client, finalFilename)
}
// Set up build environment.
sep := "/"
if b.OS == "windows" {
sep = "\\"
}
env := append(bc.Env(),
"GOROOT_FINAL="+bc.GorootFinal(),
"GOROOT="+work+sep+goDir,
"GOPATH="+work+sep+goPath,
"GOBIN=",
)
if b.Goarm > 0 {
env = append(env, fmt.Sprintf("GOARM=%d", b.Goarm))
env = append(env, fmt.Sprintf("CGO_CFLAGS=-march=armv%d", b.Goarm))
env = append(env, fmt.Sprintf("CGO_LDFLAGS=-march=armv%d", b.Goarm))
}
// Issues #36025 #35459
if b.OS == "darwin" && b.Arch == "amd64" {
minMacVersion := minSupportedMacOSVersion(*version)
env = append(env, fmt.Sprintf("CGO_CFLAGS=-mmacosx-version-min=%s", minMacVersion))
}
// Execute build (make.bash only first).
b.logf("Building (make.bash only).")
out := new(bytes.Buffer)
var execOut io.Writer = out
if *watch && *target != "" {
execOut = io.MultiWriter(out, os.Stdout)
}
remoteErr, err := client.Exec(context.Background(), filepath.Join(goDir, bc.MakeScript()), buildlet.ExecOpts{
Output: execOut,
ExtraEnv: env,
Args: bc.MakeScriptArgs(),
})
if err != nil {
return err
}
if remoteErr != nil {
return fmt.Errorf("Build failed: %v\nOutput:\n%v", remoteErr, out)
}
if err := b.checkRelocations(client); err != nil {
return err
}
goCmd := path.Join(goDir, "bin/go")
if b.OS == "windows" {
goCmd += ".exe"
}
runGo := func(args ...string) error {
out := new(bytes.Buffer)
var execOut io.Writer = out
if *watch && *target != "" {
execOut = io.MultiWriter(out, os.Stdout)
}
cmdEnv := append([]string(nil), env...)
if len(args) > 0 && args[0] == "run" && hostArch != "" {
cmdEnv = setGOARCH(cmdEnv, hostArch)
}
remoteErr, err := client.Exec(context.Background(), goCmd, buildlet.ExecOpts{
Output: execOut,
Dir: ".", // root of buildlet work directory
Args: args,
ExtraEnv: cmdEnv,
})
if err != nil {
return err
}
if remoteErr != nil {
return fmt.Errorf("go %v: %v\n%s", strings.Join(args, " "), remoteErr, out)
}
return nil
}
if b.Race {
b.logf("Building race detector.")
if err := runGo("install", "-race", "std"); err != nil {
return err
}
}
// postBuildCleanFiles are the list of files to remove in the go/ directory
// after things have been built.
postBuildCleanFiles := []string{
"VERSION.cache",
"pkg/bootstrap",
}
// Remove race detector *.syso files for other GOOS/GOARCHes (except for the source release).
if !b.Source {
okayRace := fmt.Sprintf("race_%s_%s.syso", b.OS, b.Arch)
err := client.ListDir(ctx, ".", buildlet.ListDirOpts{Recursive: true}, func(ent buildlet.DirEntry) {
name := strings.TrimPrefix(ent.Name(), "go/")
if strings.HasPrefix(name, "src/runtime/race/race_") &&
strings.HasSuffix(name, ".syso") &&
path.Base(name) != okayRace {
postBuildCleanFiles = append(postBuildCleanFiles, name)
}
})
if err != nil {
return fmt.Errorf("enumerating files to clean race syso files: %v", err)
}
}
b.logf("Cleaning goroot (post-build).")
if err := client.RemoveAll(ctx, addPrefix(goDir, postBuildCleanFiles)...); err != nil {
return err
}
// Users don't need the api checker binary pre-built. It's
// used by tests, but all.bash builds it first.
if err := client.RemoveAll(ctx, b.toolDir()+"/api"); err != nil {
return err
}
// The oldlink tool is used for debugging differences during
// testing of the linker rewrite, and can be built by users
// if necessary. See issue 39509.
// This can be removed when oldlink is gone, likely once Go 1.16
// is no longer supported.
if err := client.RemoveAll(ctx, b.toolDir()+"/oldlink"); err != nil {
return err
}
// Remove go/pkg/${GOOS}_${GOARCH}/cmd. This saves a bunch of
// space, and users don't typically rebuild cmd/compile,
// cmd/link, etc. If they want to, they still can, but they'll
// have to pay the cost of rebuilding dependent libaries. No
// need to ship them just in case.
//
// Also remove go/pkg/${GOOS}_${GOARCH}_{dynlink,shared,testcshared_shared}
// per Issue 20038.
if err := client.RemoveAll(ctx,
b.pkgDir()+"/cmd",
b.pkgDir()+"_dynlink",
b.pkgDir()+"_shared",
b.pkgDir()+"_testcshared_shared",
); err != nil {
return err
}
b.logf("Pushing and running releaselet.")
err = client.Put(ctx, strings.NewReader(releaselet), "releaselet.go", 0666)
if err != nil {
return err
}
if err := runGo("run", "releaselet.go"); err != nil {
log.Printf("releaselet failed: %v", err)
client.ListDir(ctx, ".", buildlet.ListDirOpts{Recursive: true}, func(ent buildlet.DirEntry) {
log.Printf("remote: %v", ent)
})
return err
}
cleanFiles := []string{"releaselet.go", goPath, go14, "tmp", "gocache"}
// So far, we've run make.bash. We want to create the release archive next.
// Since the release archive hasn't been tested yet, place it in a temporary
// location. After all.bash runs successfully (or gets explicitly skipped),
// we'll move the release archive to its final location. For TestOnly builds,
// we only care whether tests passed and do not produce release artifacts.
type releaseFile struct {
Untested string // Temporary location of the file before the release has been tested.
Final string // Final location where to move the file after the release has been tested.
}
var releases []releaseFile
stagingDir := *stagingDir
if stagingDir == "" {
var err error
stagingDir, err = ioutil.TempDir("", "go-release-staging_")
if err != nil {
log.Fatal(err)
}
}
stagingFile := func(ext string) string {
return filepath.Join(stagingDir, *version+"."+b.String()+ext+".untested")
}
if !b.TestOnly && b.OS == "windows" {
untested := stagingFile(".msi")
if err := b.fetchFile(client, untested, "msi"); err != nil {
return err
}
releases = append(releases, releaseFile{
Untested: untested,
Final: *version + "." + b.String() + ".msi",
})
}
if b.OS == "windows" {
cleanFiles = append(cleanFiles, "msi")
}
if b.OS == "windows" && b.Arch == "arm64" {
// At least on windows-arm64, 'wix/winterop.dll' gets created.
// Delete the entire wix directory since it's unrelated to Go.
cleanFiles = append(cleanFiles, "wix")
}
// Need to delete everything except the final "go" directory,
// as we make the tarball relative to workdir.
b.logf("Cleaning workdir.")
if err := client.RemoveAll(ctx, cleanFiles...); err != nil {
return err
}
// And verify there's no other top-level stuff besides the "go" directory:
if err := b.checkTopLevelDirs(ctx, client); err != nil {
return fmt.Errorf("verifying no unwanted top-level directories: %v", err)
}
if err := b.checkPerm(ctx, client); err != nil {
return fmt.Errorf("verifying file permissions: %v", err)
}
switch {
case !b.TestOnly && b.OS != "windows":
untested := stagingFile(".tar.gz")
if err := b.fetchTarball(ctx, client, untested); err != nil {
return fmt.Errorf("fetching and writing tarball: %v", err)
}
releases = append(releases, releaseFile{
Untested: untested,
Final: *version + "." + b.String() + ".tar.gz",
})
case !b.TestOnly && b.OS == "windows":
untested := stagingFile(".zip")
if err := b.fetchZip(client, untested); err != nil {
return fmt.Errorf("fetching and writing zip: %v", err)
}
releases = append(releases, releaseFile{
Untested: untested,
Final: *version + "." + b.String() + ".zip",
})
case b.TestOnly:
// Use an empty .test-only file to indicate the test outcome.
// This file gets moved from its initial location in the
// release-staging directory to the final release directory
// when the test-only build passes tests successfully.
untested := stagingFile(".test-only")
if err := ioutil.WriteFile(untested, nil, 0600); err != nil {
return fmt.Errorf("writing empty test-only file: %v", err)
}
releases = append(releases, releaseFile{
Untested: untested,
Final: *version + "." + b.String() + ".test-only",
})
}
// Execute build (all.bash) if running tests.
if *skipTests || b.SkipTests {
b.logf("Skipping all.bash tests.")
} else {
if u := bc.GoBootstrapURL(buildEnv); u != "" {
b.logf("Installing go1.4 (second time, for all.bash).")
if err := client.PutTarFromURL(ctx, u, go14); err != nil {
return err
}
}
b.logf("Building (all.bash to ensure tests pass).")
out := new(bytes.Buffer)
var execOut io.Writer = out
if *watch && *target != "" {
execOut = io.MultiWriter(out, os.Stdout)
}
remoteErr, err := client.Exec(ctx, filepath.Join(goDir, bc.AllScript()), buildlet.ExecOpts{
Output: execOut,
ExtraEnv: env,
Args: bc.AllScriptArgs(),
})
if err != nil {
return err
}
if remoteErr != nil {
return fmt.Errorf("Build failed: %v\nOutput:\n%v", remoteErr, out)
}
}
// If we get this far, the all.bash tests have passed (or been skipped).
// Move untested release files to their final locations.
for _, r := range releases {
b.logf("Moving %q to %q.", r.Untested, r.Final)
if err := os.Rename(r.Untested, r.Final); err != nil {
return err
}
}
return nil
}
// checkTopLevelDirs checks that all files under client's "."
// ($WORKDIR) are are under "go/".
func (b *Build) checkTopLevelDirs(ctx context.Context, client *buildlet.Client) error {
var badFileErr error // non-nil once an unexpected file/dir is found
if err := client.ListDir(ctx, ".", buildlet.ListDirOpts{Recursive: true}, func(ent buildlet.DirEntry) {
name := ent.Name()
if !(strings.HasPrefix(name, "go/") || strings.HasPrefix(name, `go\`)) {
b.logf("unexpected file: %q", name)
if badFileErr == nil {
badFileErr = fmt.Errorf("unexpected filename %q found after cleaning", name)
}
}
}); err != nil {
return err
}
return badFileErr
}
// checkPerm checks that files in client's $WORKDIR/go directory
// have expected permissions.
func (b *Build) checkPerm(ctx context.Context, client *buildlet.Client) error {
var badPermErr error // non-nil once an unexpected perm is found
checkPerm := func(ent buildlet.DirEntry, allowed ...string) {
for _, p := range allowed {
if ent.Perm() == p {
return
}
}
b.logf("unexpected file %q perm: %q", ent.Name(), ent.Perm())
if badPermErr == nil {
badPermErr = fmt.Errorf("unexpected file %q perm %q found", ent.Name(), ent.Perm())
}
}
if err := client.ListDir(ctx, "go", buildlet.ListDirOpts{Recursive: true}, func(ent buildlet.DirEntry) {
switch b.OS {
default:
checkPerm(ent, "drwxr-xr-x", "-rw-r--r--", "-rwxr-xr-x")
case "windows":
checkPerm(ent, "drwxrwxrwx", "-rw-rw-rw-")
}
}); err != nil {
return err
}
if !b.Source {
if err := client.ListDir(ctx, "go/bin", buildlet.ListDirOpts{}, func(ent buildlet.DirEntry) {
switch b.OS {
default:
checkPerm(ent, "-rwxr-xr-x")
case "windows":
checkPerm(ent, "-rw-rw-rw-")
}
}); err != nil {
return err
}
}
return badPermErr
}
func (b *Build) fetchTarball(ctx context.Context, client *buildlet.Client, dest string) error {
b.logf("Downloading tarball.")
tgz, err := client.GetTar(ctx, ".")
if err != nil {
return err
}
return b.writeFile(dest, tgz)
}
func (b *Build) fetchZip(client *buildlet.Client, dest string) error {
b.logf("Downloading tarball and re-compressing as zip.")
tgz, err := client.GetTar(context.Background(), ".")
if err != nil {
return err
}
defer tgz.Close()
f, err := os.Create(dest)
if err != nil {
return err
}
if err := tgzToZip(f, tgz); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
b.logf("Wrote %q.", dest)
return nil
}
func tgzToZip(w io.Writer, r io.Reader) error {
zr, err := gzip.NewReader(r)
if err != nil {
return err
}
tr := tar.NewReader(zr)
zw := zip.NewWriter(w)
for {
th, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
fi := th.FileInfo()
zh, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
zh.Name = th.Name // for the full path
switch strings.ToLower(path.Ext(zh.Name)) {
case ".jpg", ".jpeg", ".png", ".gif":
// Don't re-compress already compressed files.
zh.Method = zip.Store
default:
zh.Method = zip.Deflate
}
if fi.IsDir() {
zh.Method = zip.Store
}
w, err := zw.CreateHeader(zh)
if err != nil {
return err
}
if fi.IsDir() {
continue
}
if _, err := io.Copy(w, tr); err != nil {
return err
}
}
return zw.Close()
}
// fetchFile fetches the specified directory from the given buildlet, and
// writes the first file it finds in that directory to dest.
func (b *Build) fetchFile(client *buildlet.Client, dest, dir string) error {
b.logf("Downloading file from %q.", dir)
tgz, err := client.GetTar(context.Background(), dir)
if err != nil {
return err
}
defer tgz.Close()
zr, err := gzip.NewReader(tgz)
if err != nil {
return err
}
tr := tar.NewReader(zr)
for {
h, err := tr.Next()
if err == io.EOF {
return io.ErrUnexpectedEOF
}
if err != nil {
return err
}
if !h.FileInfo().IsDir() {
break
}
}
return b.writeFile(dest, tr)
}
func (b *Build) writeFile(name string, r io.Reader) error {
f, err := os.Create(name)
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
if strings.HasSuffix(name, ".gz") {
if err := verifyGzipSingleStream(name); err != nil {
return fmt.Errorf("error verifying that %s is a single-stream gzip: %v", name, err)
}
}
b.logf("Wrote %q.", name)
return nil
}
// checkRelocations runs readelf on pkg/linux_amd64/runtime/cgo.a and makes sure
// we don't see R_X86_64_REX_GOTPCRELX in new Go 1.15 minor releases.
// See golang.org/issue/31293 and golang.org/issue/40561#issuecomment-731482962.
func (b *Build) checkRelocations(client *buildlet.Client) error {
if b.OS != "linux" || b.Arch != "amd64" || b.TestOnly {
// This check is only applicable to linux/amd64 builds.
// However, skip it on test-only builds because they
// don't produce binaries that are shipped to users.
return nil
}
var out bytes.Buffer
file := fmt.Sprintf("go/pkg/linux_%s/runtime/cgo.a", b.Arch)
remoteErr, err := client.Exec(context.Background(), "readelf", buildlet.ExecOpts{
Output: &out,
Args: []string{"-r", "--wide", file},
SystemLevel: true, // look for readelf in system's PATH
})
if err != nil {
return fmt.Errorf("failed to run readelf: %v", err)
}
got := out.String()
switch {
default: // Go 1.16 and newer.
// Note: This check was kept and updated for Go 1.16, since it wasn't hard.
// Remove it at some point in the future if it becomes no longer useful or
// overly expensive to maintain.
if strings.Contains(got, "R_X86_64_GOTPCREL") {
return fmt.Errorf("%s contained a R_X86_64_GOTPCREL relocation", file)
}
if !strings.Contains(got, "R_X86_64_REX_GOTPCRELX") {
return fmt.Errorf("%s did not contain a R_X86_64_REX_GOTPCRELX relocation; remoteErr=%v, %s", file, remoteErr, got)
}
case strings.HasPrefix(*version, "go1.15"):
if strings.Contains(got, "R_X86_64_REX_GOTPCRELX") {
return fmt.Errorf("%s contained a R_X86_64_REX_GOTPCRELX relocation", file)
}
if !strings.Contains(got, "R_X86_64_GOTPCREL") {
return fmt.Errorf("%s did not contain a R_X86_64_GOTPCREL relocation; remoteErr=%v, %s", file, remoteErr, got)
}
}
return nil
}
// verifyGzipSingleStream verifies that the named gzip file is not
// a multi-stream file. See golang.org/issue/19052
func verifyGzipSingleStream(name string) error {
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
br := bufio.NewReader(f)
zr, err := gzip.NewReader(br)
if err != nil {
return err
}
zr.Multistream(false)
if _, err := io.Copy(ioutil.Discard, zr); err != nil {
return fmt.Errorf("reading first stream: %v", err)
}
peek, err := br.Peek(1)
if len(peek) > 0 || err != io.EOF {
return fmt.Errorf("unexpected peek of %d, %v after first gzip stream", len(peek), err)
}
return nil
}
func addPrefix(prefix string, in []string) []string {
var out []string
for _, s := range in {
out = append(out, path.Join(prefix, s))
}
return out
}
func coordinatorClient() *buildlet.CoordinatorClient {
return &buildlet.CoordinatorClient{
Auth: buildlet.UserPass{
Username: "user-" + *user,
Password: userToken(),
},
Instance: build.ProdCoordinator,
}
}
func homeDir() string {
if runtime.GOOS == "windows" {
return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
}
return os.Getenv("HOME")
}
func configDir() string {
if runtime.GOOS == "windows" {
return filepath.Join(os.Getenv("APPDATA"), "Gomote")
}
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
return filepath.Join(xdg, "gomote")
}
return filepath.Join(homeDir(), ".config", "gomote")
}
func username() string {
if runtime.GOOS == "windows" {
return os.Getenv("USERNAME")
}
return os.Getenv("USER")
}
func userToken() string {
if *user == "" {
panic("userToken called with user flag empty")
}
keyDir := configDir()
baseFile := "user-" + *user + ".token"
tokenFile := filepath.Join(keyDir, baseFile)
slurp, err := ioutil.ReadFile(tokenFile)
if os.IsNotExist(err) {
log.Printf("Missing file %s for user %q. Change --user or obtain a token and place it there.",
tokenFile, *user)
}
if err != nil {
log.Fatal(err)
}
return strings.TrimSpace(string(slurp))
}
func setGOARCH(env []string, goarch string) []string {
wantKV := "GOARCH=" + goarch
existing := false
for i, kv := range env {
if strings.HasPrefix(kv, "GOARCH=") && kv != wantKV {
env[i] = wantKV
existing = true
}
}
if existing {
return env
}
return append(env, wantKV)
}
// minSupportedMacOSVersion provides the minimum supported macOS
// version (of the form N.M) for supported Go versions.
func minSupportedMacOSVersion(goVer string) string {
// TODO(amedee,dmitshur,golang.org/issue/40558): Use a version package to compare versions of Go.
// The minimum supported version of macOS with each version of go:
// go1.15 - macOS 10.12
// go1.16 - macOS 10.12
// go1.17 - macOS 10.13
minMacVersion := "10.13"
if match("< go1.17beta1", goVer) {
minMacVersion = "10.12"
return minMacVersion
}
return minMacVersion
}
// match reports whether the Go version goVer matches the provided version query.
// The empty query matches all Go versions.
// match panics if given a query that it doesn't support.
func match(query, goVer string) bool {
// TODO(golang.org/issue/40558): This should help inform the API for a Go version parser.
switch query {
case "": // A special case to make the zero Build.GoQuery value useful.
return true
case ">= go1.17beta1":
return !strings.HasPrefix(goVer, "go1.16") && !strings.HasPrefix(goVer, "go1.15")
case "< go1.17beta1":
return strings.HasPrefix(goVer, "go1.16") || strings.HasPrefix(goVer, "go1.15")
case ">= go1.16beta1":
return !strings.HasPrefix(goVer, "go1.15")
case "< go1.16beta1":
return strings.HasPrefix(goVer, "go1.15")
default:
panic(fmt.Errorf("match: query %q is not supported", query))
}
}