blob: 9af46183cc53ebf806436fd5e1dc5d93252b4750 [file] [log] [blame]
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// The upload command writes a file to Google Cloud Storage. It's used
// exclusively by the Makefiles in the Go project repos. Think of it
// as a very light version of gsutil or gcloud, but with some
// Go-specific configuration knowledge baked in.
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"regexp"
"strings"
"time"
"cloud.google.com/go/storage"
"golang.org/x/build/internal/envutil"
)
var (
public = flag.Bool("public", false, "object should be world-readable")
cacheable = flag.Bool("cacheable", true, "object should be cacheable")
file = flag.String("file", "-", "Filename to read object from, or '-' for stdin. If it begins with 'go:' then the rest is considered to be a Go target to install first, and then upload.")
verbose = flag.Bool("verbose", false, "verbose logging")
osarch = flag.String("osarch", "", "Optional 'GOOS-GOARCH' value to cross-compile; used only if --file begins with 'go:'. As a special case, if the value contains a '.' byte, anything up to and including that period is discarded.")
project = flag.String("project", "", "GCE Project. If blank, it's automatically inferred from the bucket name for the common Go buckets.")
tags = flag.String("tags", "", "tags to pass to go list, go install, etc. Only applicable if the --file value begins with 'go:'")
doGzip = flag.Bool("gzip", false, "gzip the stored contents (not the upload's Content-Encoding); this forces the Content-Type to be application/octet-stream. To prevent misuse, the object name must also end in '.gz'")
extraEnv = flag.String("extraenv", "", "comma-separated list of addition KEY=val environment pairs to include in build environment when building a target to upload")
installSuffix = flag.String("installsuffix", "", "installsuffix for the go command")
static = flag.Bool("static", false, "compile the binary statically, adds necessary ldflags")
goVer = flag.String("go", "", "optional Go version to use for compilation when using the -file flag to build a Go binary; the Go version is fetched as needed")
)
// to match uploads to e.g. https://storage.googleapis.com/golang/go1.4-bootstrap-20170531.tar.gz.
var go14BootstrapRx = regexp.MustCompile(`^go1\.4-bootstrap-20\d{6}\.tar\.gz$`)
func main() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `Usage: upload [--public] [--file=...] <bucket/object>
If <bucket/object> is of the form "golang/go1.4-bootstrap-20yymmdd.tar.gz",
then the current release-branch.go1.4 is uploaded from Gerrit, with each
tar entry filename beginning with the prefix "go/".
`)
flag.PrintDefaults()
}
flag.Parse()
if flag.NArg() != 1 {
flag.Usage()
os.Exit(1)
}
args := strings.SplitN(flag.Arg(0), "/", 2)
if len(args) != 2 {
flag.Usage()
os.Exit(1)
}
if strings.HasPrefix(*file, "go:") {
buildGoTarget()
}
bucket, object := args[0], args[1]
// Special support for auto-tarring up Go 1.4 tarballs from the 1.4 release branch.
is14Src := bucket == "golang" && go14BootstrapRx.MatchString(object)
if is14Src {
if *file != "-" {
log.Fatalf("invalid use of --file with Go 1.4 tarball %v", object)
}
*doGzip = true
*public = true
*cacheable = true
}
if *doGzip && !strings.HasSuffix(object, ".gz") {
log.Fatalf("--gzip flag requires object ending in .gz")
}
proj := *project
if proj == "" {
proj, _ = bucketProject[bucket]
if proj == "" {
log.Fatalf("bucket %q doesn't have an associated project in upload.go", bucket)
}
}
ctx := context.Background()
storageClient, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("storage.NewClient: %v", err)
}
if is14Src {
_, err := storageClient.Bucket(bucket).Object(object).Attrs(context.Background())
if err != storage.ErrObjectNotExist {
if err == nil {
log.Fatalf("object %v already exists; refusing to overwrite.", object)
}
log.Fatalf("error checking for %v: %v", object, err)
}
} else if alreadyUploaded(storageClient, bucket, object) {
if *verbose {
log.Printf("Already uploaded.")
}
return
}
w := storageClient.Bucket(bucket).Object(object).NewWriter(ctx)
// If you don't give the owners access, the web UI seems to
// have a bug and doesn't have access to see that it's public, so
// won't render the "Shared Publicly" link. So we do that, even
// though it's dumb and unnecessary otherwise:
w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.ACLEntity("project-owners-" + proj), Role: storage.RoleOwner})
if *public {
w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader})
if !*cacheable {
w.CacheControl = "no-cache"
}
}
var content io.Reader
switch {
case is14Src:
content = generate14Tarfile()
case *file == "-":
content = os.Stdin
default:
content, err = os.Open(*file)
if err != nil {
log.Fatal(err)
}
}
if *doGzip {
var zbuf bytes.Buffer
zw := gzip.NewWriter(&zbuf)
if _, err := io.Copy(zw, content); err != nil {
log.Fatalf("compressing content: %v", err)
}
if err := zw.Close(); err != nil {
log.Fatalf("gzip.Close: %v", err)
}
content = &zbuf
}
const maxSlurp = 1 << 20
var buf bytes.Buffer
n, err := io.CopyN(&buf, content, maxSlurp)
if err != nil && err != io.EOF {
log.Fatalf("Error reading from stdin: %v, %v", n, err)
}
if *doGzip {
w.ContentType = "application/octet-stream"
} else {
w.ContentType = http.DetectContentType(buf.Bytes())
}
_, err = io.Copy(w, io.MultiReader(&buf, content))
if cerr := w.Close(); cerr != nil && err == nil {
err = cerr
}
if err != nil {
log.Fatalf("Write error: %v", err)
}
if *verbose {
log.Printf("Uploaded %v", object)
}
os.Exit(0)
}
var bucketProject = map[string]string{
"dev-gccgo-builder-data": "gccgo-dashboard-dev",
"dev-go-builder-data": "go-dashboard-dev",
"gccgo-builder-data": "gccgo-dashboard-builders",
"go-builder-data": "symbolic-datum-552",
"go-build-log": "symbolic-datum-552",
"http2-demo-server-tls": "symbolic-datum-552",
"winstrap": "999119582588",
"gobuilder": "999119582588", // deprecated
"golang": "999119582588",
}
func buildGoTarget() {
target := strings.TrimPrefix(*file, "go:")
var goos, goarch string
if *osarch != "" {
*osarch = strings.TrimSuffix(*osarch, ".gz")
*osarch = (*osarch)[strings.LastIndex(*osarch, ".")+1:]
v := strings.Split(*osarch, "-")
if len(v) == 3 {
v = v[:2] // support e.g. "linux-arm-aws" as GOOS=linux, GOARCH=arm
}
if len(v) != 2 || v[0] == "" || v[1] == "" {
log.Fatalf("invalid -osarch value %q", *osarch)
}
goos, goarch = v[0], v[1]
}
goBin := goCmd()
env := []string{"GOOS=" + goos, "GOARCH=" + goarch}
if *extraEnv != "" {
env = append(env, strings.Split(*extraEnv, ",")...)
}
cmd := exec.Command(goBin,
"list",
"--tags="+*tags,
"--installsuffix="+*installSuffix,
"-f", "{{.Target}}",
target)
envutil.SetEnv(cmd, env...)
out, err := cmd.Output()
if err != nil {
log.Fatalf("go list: %v", err)
}
outFile := string(bytes.TrimSpace(out))
fi0, err := os.Stat(outFile)
if os.IsNotExist(err) {
if *verbose {
log.Printf("File %s doesn't exist; building...", outFile)
}
}
version := os.Getenv("USER") + "-" + time.Now().Format(time.RFC3339)
ldflags := "-X main.Version=" + version
if *static {
ldflags = "-linkmode=external -extldflags '-static -pthread' " + ldflags
}
cmd = exec.Command(goBin,
"install",
"--tags="+*tags,
"--installsuffix="+*installSuffix,
"-x",
"--ldflags="+ldflags,
target)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if *verbose {
cmd.Stdout = os.Stdout
}
envutil.SetEnv(cmd, env...)
if err := cmd.Run(); err != nil {
log.Fatalf("go install %s: %v, %s", target, err, stderr.Bytes())
}
fi1, err := os.Stat(outFile)
if err != nil {
log.Fatalf("Expected output file %s stat failure after go install %v: %v", outFile, target, err)
}
if !os.SameFile(fi0, fi1) {
if *verbose {
log.Printf("File %s rebuilt.", outFile)
}
}
*file = outFile
}
func goCmd() string {
if *goVer == "" {
return "go"
}
v := "go" + strings.TrimPrefix(*goVer, "go")
p, err := exec.LookPath(v)
if err != nil {
log.Printf("%s not found in path; running go get golang.org/dl/%v", v, v)
out, err := exec.Command("go", "get", "golang.org/dl/"+v).CombinedOutput()
if err != nil {
log.Fatalf("%v; %s", err, out)
}
p, err = exec.LookPath(v)
if err != nil {
log.Fatalf("%s still not in $PATH after go get golang.org/dl/%v", v, v)
}
}
out, err := exec.Command(p, "download").CombinedOutput()
if err != nil {
log.Fatalf("downloading %v: %v, %s", *goVer, err, out)
}
return p
}
// alreadyUploaded reports whether *file has already been uploaded and the correct contents
// are on cloud storage already.
func alreadyUploaded(storageClient *storage.Client, bucket, object string) bool {
if *file == "-" {
return false // don't know.
}
o, err := storageClient.Bucket(bucket).Object(object).Attrs(context.Background())
if err == storage.ErrObjectNotExist {
return false
}
if err != nil {
log.Printf("Warning: stat failure: %v", err)
return false
}
m5 := md5.New()
fi, err := os.Stat(*file)
if err != nil {
log.Fatal(err)
}
if fi.Size() != o.Size {
return false
}
f, err := os.Open(*file)
if err != nil {
log.Fatal(err)
}
defer f.Close()
n, err := io.Copy(m5, f)
if err != nil {
log.Fatal(err)
}
if n != fi.Size() {
log.Printf("Warning: file size of %v changed", *file)
}
return bytes.Equal(m5.Sum(nil), o.MD5)
}
// generate14Tarfile downloads the release-branch.go1.4 release branch
// tarball and returns it uncompressed, with the "go/" prefix before
// each tar header's filename.
func generate14Tarfile() io.Reader {
const tarURL = "https://go.googlesource.com/go/+archive/release-branch.go1.4.tar.gz"
res, err := http.Get(tarURL)
if err != nil {
log.Fatal(err)
}
if res.StatusCode != 200 {
log.Fatalf("%v: %v", tarURL, res.Status)
}
if got, want := res.Header.Get("Content-Type"), "application/x-gzip"; got != want {
log.Fatalf("%v: response Content-Type = %q; expected %q", tarURL, got, want)
}
var out bytes.Buffer // output tar (not gzipped)
tw := tar.NewWriter(&out)
zr, err := gzip.NewReader(res.Body)
if err != nil {
log.Fatal(err)
}
tr := tar.NewReader(zr)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
switch hdr.Typeflag {
case tar.TypeReg, tar.TypeRegA, tar.TypeSymlink, tar.TypeDir:
// Accept these.
default:
continue
}
hdr.Name = "go/" + hdr.Name
if err := tw.WriteHeader(hdr); err != nil {
log.Fatalf("WriteHeader: %v", err)
}
if _, err := io.Copy(tw, tr); err != nil {
log.Fatalf("tar copying %v: %v", hdr.Name, err)
}
}
if err := tw.Close(); err != nil {
log.Fatal(err)
}
return &out
}