blob: 3ff7ab1799ab959ae0c5d8e2f7589a6fc5b31744 [file] [log] [blame]
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Genbootstrap prepares GOROOT_BOOTSTRAP tarballs suitable for
use on builders. It's a wrapper around bootstrap.bash. After
bootstrap.bash produces the full output, genbootstrap trims it up,
removing unnecessary and unwanted files.
Usage:
genbootstrap [-upload] [-rev=rev] [-v] GOOS-GOARCH[-suffix]...
The argument list can be a single glob pattern (for example '*'),
which expands to all known targets matching that pattern.
Deprecated: As of Go 1.21.0, genbootstrap is superseded
by make.bash -distpack and doesn't need to be used anymore.
The one exception are GOOS=windows targets, since their
go.dev/dl downloads are in .zip format but the builders
in x/build support pushing .tar.gz format only.
*/
package main
import (
"bytes"
"context"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strings"
"cloud.google.com/go/storage"
"golang.org/x/build/dashboard"
"golang.org/x/build/internal/envutil"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
)
var skipBuild = flag.String("skip_build", "", "skip bootstrap.bash and reuse output in `dir` instead")
var upload = flag.Bool("upload", false, "upload outputs to gs://go-builder-data/")
var verbose = flag.Bool("v", false, "show verbose output")
var rev = flag.String("rev", "go1.17.13", "build Go at Git revision `rev`")
func usage() {
fmt.Fprintln(os.Stderr, "Usage: genbootstrap GOOS-GOARCH[-GO$GOARCH]... (or a glob pattern like '*')")
flag.PrintDefaults()
}
func main() {
flag.Usage = usage
flag.Parse()
if flag.NArg() < 1 {
flag.Usage()
os.Exit(2)
}
list := flag.Args()
if len(list) == 1 && strings.ContainsAny(list[0], "*?[]") {
pattern := list[0]
list = nil
for _, name := range allPairs() {
if ok, err := path.Match(pattern, name); ok {
list = append(list, name)
} else if err != nil {
log.Fatalf("invalid match: %v", err)
}
}
if len(list) == 0 {
log.Fatalf("no matches for %q", pattern)
}
log.Printf("expanded %s: %v", pattern, list)
}
var nonWindowsTargets []string
for _, pair := range list {
f := strings.Split(pair, "-")
if len(f) != 2 && len(f) != 3 {
log.Fatalf("invalid target: %q", pair)
}
if goos := f[0]; goos != "windows" {
nonWindowsTargets = append(nonWindowsTargets, pair)
}
}
if x, _ := version.Go1PointX(*rev); x >= 21 && len(nonWindowsTargets) > 0 {
log.Fatalf("genbootstrap isn't needed to build Go 1.21.0 and newer bootstrap toolchains for %v, "+
"they're already built with make.bash -distpack and made available at go.dev/dl for all ports "+
"(GOOS=windows targets are permitted; builders need .tar.gz format so go.dev/dl can't be used as is)", nonWindowsTargets)
}
dir, err := os.MkdirTemp("", "genbootstrap-*")
if err != nil {
log.Fatal(err)
}
goroot := filepath.Join(dir, "goroot")
if err := os.MkdirAll(goroot, 0777); err != nil {
log.Fatal(err)
}
log.Printf("Bootstrapping in %s at revision %s\n", dir, *rev)
resp, err := http.Get("https://go.googlesource.com/go/+archive/" + *rev + ".tar.gz")
if err != nil {
log.Fatal(err)
}
if resp.StatusCode != 200 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
log.Fatalf("fetching %s: %v\n%s", *rev, resp.Status, body)
}
cmd := exec.Command("tar", "-C", goroot, "-xzf", "-")
cmd.Stdin = resp.Body
out, err := cmd.CombinedOutput()
if err != nil {
log.Fatalf("tar: %v\n%s", err, out)
}
// Work around GO_LDSO bug by removing implicit setting from make.bash.
// See go.dev/issue/54196 and go.dev/issue/54197.
// Even if those are fixed, the old toolchains we are using for bootstrap won't get the fix.
makebash := filepath.Join(goroot, "src/make.bash")
data, err := os.ReadFile(makebash)
if err != nil {
log.Fatal(err)
}
data = bytes.ReplaceAll(data, []byte("GO_LDSO"), []byte("GO_LDSO_BUG"))
if err := os.WriteFile(makebash, data, 0666); err != nil {
log.Fatal(err)
}
var storageClient *storage.Client
if *upload {
ctx := context.Background()
storageClient, err = storage.NewClient(ctx)
if err != nil {
log.Fatalf("storage.NewClient: %v", err)
}
}
gorootSrc := filepath.Join(goroot, "src")
List:
for _, pair := range list {
f := strings.Split(pair, "-")
goos, goarch, gosuffix := f[0], f[1], ""
if len(f) == 3 {
gosuffix = "-" + f[2]
}
log.Printf("# %s-%s%s\n", goos, goarch, gosuffix)
tgz := filepath.Join(dir, "gobootstrap-"+goos+"-"+goarch+gosuffix+"-"+*rev+".tar.gz")
os.Remove(tgz)
outDir := filepath.Join(dir, "go-"+goos+"-"+goarch+"-bootstrap")
if *skipBuild != "" {
outDir = *skipBuild
} else {
os.RemoveAll(outDir)
cmd := exec.Command(filepath.Join(gorootSrc, "bootstrap.bash"))
envutil.SetDir(cmd, gorootSrc)
envutil.SetEnv(cmd,
"GOROOT="+goroot,
"CGO_ENABLED=0",
"GOOS="+goos,
"GOARCH="+goarch,
"GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
)
if gosuffix != "" {
envutil.SetEnv(cmd, "GO"+strings.ToUpper(goarch)+"="+gosuffix[len("-"):])
}
if *verbose {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
log.Print(err)
continue List
}
} else {
if out, err := cmd.CombinedOutput(); err != nil {
os.Stdout.Write(out)
log.Print(err)
continue List
}
}
// bootstrap.bash makes a bzipped tar file too,
// but it's fat and full of stuff we don't need. Delete it.
os.Remove(outDir + ".tbz")
}
if err := filepath.Walk(outDir, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
rel := strings.TrimPrefix(strings.TrimPrefix(path, outDir), "/")
base := filepath.Base(path)
var pkgrel string // relative to pkg/<goos>_<goarch>/, or empty
if strings.HasPrefix(rel, "pkg/") && strings.Count(rel, "/") >= 2 {
pkgrel = strings.TrimPrefix(rel, "pkg/")
pkgrel = pkgrel[strings.Index(pkgrel, "/")+1:]
if *verbose {
log.Printf("rel %q => %q", rel, pkgrel)
}
}
remove := func() error {
if err := os.RemoveAll(path); err != nil {
return err
}
if fi.IsDir() {
return filepath.SkipDir
}
return nil
}
switch pkgrel {
case "cmd":
return remove()
}
switch rel {
case "api",
"bin/gofmt",
"doc",
"misc/android",
"misc/cgo",
"misc/chrome",
"misc/swig",
"test":
return remove()
}
if base == "testdata" {
return remove()
}
if strings.HasPrefix(rel, "pkg/tool/") {
switch base {
case "addr2line", "api", "cgo", "cover",
"dist", "doc", "fix", "nm",
"objdump", "pack", "pprof",
"trace", "vet", "yacc":
return remove()
}
}
if fi.IsDir() {
return nil
}
if isEditorJunkFile(path) {
return remove()
}
if !fi.Mode().IsRegular() {
return remove()
}
if strings.HasSuffix(path, "_test.go") {
return remove()
}
if *verbose {
log.Printf("keeping: %s\n", rel)
}
return nil
}); err != nil {
log.Print(err)
continue List
}
cmd := exec.Command("tar", "zcf", tgz, ".")
envutil.SetDir(cmd, outDir)
if *verbose {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
log.Print(err)
continue List
}
} else {
if out, err := cmd.CombinedOutput(); err != nil {
os.Stdout.Write(out)
log.Print(err)
continue List
}
}
log.Printf("Built %s", tgz)
if *upload {
project := "symbolic-datum-552"
bucket := "go-builder-data"
object := filepath.Base(tgz)
w := storageClient.Bucket(bucket).Object(object).NewWriter(context.Background())
// If you don't give the owners access, the web UI seems to
// have a bug and doesn't have access to see that it's public, so
// won't render the "Shared Publicly" link. So we do that, even
// though it's dumb and unnecessary otherwise:
w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.ACLEntity("project-owners-" + project), Role: storage.RoleOwner})
w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader})
f, err := os.Open(tgz)
if err != nil {
log.Print(err)
continue List
}
w.ContentType = "application/octet-stream"
_, err1 := io.Copy(w, f)
f.Close()
err = w.Close()
if err == nil {
err = err1
}
if err != nil {
log.Printf("Failed to upload %s: %v", tgz, err)
continue List
}
log.Printf("Uploaded gs://%s/%s", bucket, object)
}
}
}
func isEditorJunkFile(path string) bool {
path = filepath.Base(path)
if strings.HasPrefix(path, "#") && strings.HasSuffix(path, "#") {
return true
}
if strings.HasSuffix(path, "~") {
return true
}
return false
}
// allPairs returns a list of all known builder GOOS/GOARCH pairs.
func allPairs() []string {
have := make(map[string]bool)
var list []string
add := func(name string) {
if !have[name] {
have[name] = true
list = append(list, name)
}
}
for _, b := range dashboard.Builders {
f := strings.Split(b.Name, "-")
switch f[0] {
case "android", "ios", "js", "misc":
// skip
continue
}
name := f[0] + "-" + f[1]
if f[1] == "arm" {
add(name + "-5")
add(name + "-7")
continue
}
add(name)
}
sort.Strings(list)
return list
}