internal/task/buildrelease.go

// Copyright 2022 Go Authors All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package task

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"context"
	"embed"
	"fmt"
	"io"
	"net/http"
	"path"
	"regexp"
	"strings"
	"text/template"
	"time"

	"golang.org/x/build/buildenv"
	"golang.org/x/build/buildlet"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/internal/releasetargets"
	"golang.org/x/build/internal/workflow"
)

// WriteSourceArchive writes a source archive to out, based on revision with version written in as VERSION.
func WriteSourceArchive(ctx *workflow.TaskContext, client *http.Client, gerritURL, revision, version string, out io.Writer) error {
	ctx.Printf("Create source archive.")
	tarURL := gerritURL + "/+archive/" + revision + ".tar.gz"
	resp, err := client.Get(tarURL)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to fetch %q: %v", tarURL, resp.Status)
	}
	defer resp.Body.Close()
	gzReader, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	reader := tar.NewReader(gzReader)

	gzWriter := gzip.NewWriter(out)
	writer := tar.NewWriter(gzWriter)

	// Add go/VERSION to the archive, and fix up the existing contents.
	if err := writer.WriteHeader(&tar.Header{
		Name:       "go/VERSION",
		Size:       int64(len(version)),
		Typeflag:   tar.TypeReg,
		Mode:       0644,
		ModTime:    time.Now(),
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	}); err != nil {
		return err
	}
	if _, err := writer.Write([]byte(version)); err != nil {
		return err
	}
	if err := adjustTar(reader, writer, "go/", []adjustFunc{
		dropRegexpMatches([]string{`VERSION`}), // Don't overwrite our VERSION file from above.
		dropRegexpMatches(dropPatterns),
		fixPermissions(),
	}); err != nil {
		return err
	}

	if err := writer.Close(); err != nil {
		return err
	}
	return gzWriter.Close()
}

// An adjustFunc updates a tar file header in some way.
// The input is safe to write to. A nil return means to drop the file.
type adjustFunc func(*tar.Header) *tar.Header

// adjustTar copies the files from reader to writer, putting them in prefixDir
// and adjusting them with adjusts along the way. Prefix must have a trailing /.
func adjustTar(reader *tar.Reader, writer *tar.Writer, prefixDir string, adjusts []adjustFunc) error {
	if !strings.HasSuffix(prefixDir, "/") {
		return fmt.Errorf("prefix dir %q must have a trailing /", prefixDir)
	}
	writer.WriteHeader(&tar.Header{
		Name:       prefixDir,
		Typeflag:   tar.TypeDir,
		Mode:       0755,
		ModTime:    time.Now(),
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	})
file:
	for {
		header, err := reader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		headerCopy := *header
		newHeader := &headerCopy
		for _, adjust := range adjusts {
			newHeader = adjust(newHeader)
			if newHeader == nil {
				continue file
			}
		}
		newHeader.Name = prefixDir + newHeader.Name
		writer.WriteHeader(newHeader)
		if _, err := io.Copy(writer, reader); err != nil {
			return err
		}
	}
	return nil
}

var dropPatterns = []string{
	// .gitattributes, .github, etc.
	`\..*`,
	// This shouldn't exist, since we create a VERSION file.
	`VERSION.cache`,
	// Remove the build cache that the toolchain build process creates.
	// According to go.dev/cl/82095, it shouldn't exist at all.
	`pkg/obj/.*`,
	// Users don't need the api checker binary pre-built. It's
	// used by tests, but all.bash builds it first.
	`pkg/tool/[^/]+/api.*`,
	// Users also don't need the metadata command, which is run dynamically
	// by cmd/dist. As of writing we don't know why it's showing up at all.
	`pkg/tool/[^/]+/metadata.*`,
	// Remove pkg/${GOOS}_${GOARCH}/cmd. This saves a bunch of
	// space, and users don't typically rebuild cmd/compile,
	// cmd/link, etc. If they want to, they still can, but they'll
	// have to pay the cost of rebuilding dependent libaries. No
	// need to ship them just in case.
	`pkg/[^/]+/cmd/.*`,
	// Clean up .exe~ files; see go.dev/issue/23894.
	`.*\.exe~`,
}

// dropRegexpMatches drops files whose name matches any of patterns.
func dropRegexpMatches(patterns []string) adjustFunc {
	var rejectRegexps []*regexp.Regexp
	for _, pattern := range patterns {
		rejectRegexps = append(rejectRegexps, regexp.MustCompile("^"+pattern+"$"))
	}
	return func(h *tar.Header) *tar.Header {
		for _, regexp := range rejectRegexps {
			if regexp.MatchString(h.Name) {
				return nil
			}
		}
		return h
	}
}

// dropUnwantedSysos drops race detector sysos for other architectures.
func dropUnwantedSysos(target *releasetargets.Target) adjustFunc {
	raceSysoRegexp := regexp.MustCompile(`^src/runtime/race/race_(.*?).syso$`)
	osarch := target.GOOS + "_" + target.GOARCH
	return func(h *tar.Header) *tar.Header {
		matches := raceSysoRegexp.FindStringSubmatch(h.Name)
		if matches != nil && matches[1] != osarch {
			return nil
		}
		return h
	}
}

// fixPermissions sets files' permissions to user-writeable, world-readable.
func fixPermissions() adjustFunc {
	return func(h *tar.Header) *tar.Header {
		if h.Typeflag == tar.TypeDir || h.Mode&0111 != 0 {
			h.Mode = 0755
		} else {
			h.Mode = 0644
		}
		return h
	}
}

// fixupCrossCompile moves cross-compiled tools to their final location and
// drops unnecessary host architecture files.
func fixupCrossCompile(target *releasetargets.Target) adjustFunc {
	if !strings.HasSuffix(target.Builder, "-crosscompile") {
		return func(h *tar.Header) *tar.Header { return h }
	}
	osarch := target.GOOS + "_" + target.GOARCH
	return func(h *tar.Header) *tar.Header {
		// Move cross-compiled tools up to bin/, and drop the existing contents.
		if strings.HasPrefix(h.Name, "bin/") {
			if strings.HasPrefix(h.Name, "bin/"+osarch) {
				h.Name = strings.ReplaceAll(h.Name, "bin/"+osarch, "bin")
			} else {
				return nil
			}
		}
		// Drop host architecture files.
		if strings.HasPrefix(h.Name, "pkg/linux_amd64") ||
			strings.HasPrefix(h.Name, "pkg/tool/linux_amd64") {
			return nil
		}
		return h
	}
}

const (
	goDir = "go"
	go14  = "go1.4"
)

type BuildletStep struct {
	Target      *releasetargets.Target
	Buildlet    buildlet.RemoteClient
	BuildConfig *dashboard.BuildConfig
	LogWriter   io.Writer
}

// BuildBinary builds a binary distribution from sourceArchive and writes it to out.
func (b *BuildletStep) BuildBinary(ctx *workflow.TaskContext, sourceArchive io.Reader, out io.Writer) error {
	buildEnv := buildenv.Production
	// Push source to buildlet.
	ctx.Printf("Pushing source to buildlet.")
	if err := b.Buildlet.PutTar(ctx, sourceArchive, ""); err != nil {
		return fmt.Errorf("failed to put generated source tarball: %v", err)
	}

	if u := b.BuildConfig.GoBootstrapURL(buildEnv); u != "" {
		ctx.Printf("Installing go1.4.")
		if err := b.Buildlet.PutTarFromURL(ctx, u, go14); err != nil {
			return err
		}
	}

	// Execute build (make.bash only first).
	ctx.Printf("Building (make.bash only).")
	if err := b.exec(ctx, goDir+"/"+b.BuildConfig.MakeScript(), b.BuildConfig.MakeScriptArgs(), buildlet.ExecOpts{
		ExtraEnv: b.makeEnv(),
	}); err != nil {
		return err
	}

	if b.Target.Race {
		ctx.Printf("Building race detector.")
		if err := b.runGo(ctx, []string{"install", "-race", "std"}, buildlet.ExecOpts{
			ExtraEnv: b.makeEnv(),
		}); err != nil {
			return err
		}
	}

	ctx.Printf("Building release tarball.")
	input, err := b.Buildlet.GetTar(ctx, "go")
	if err != nil {
		return err
	}
	defer input.Close()

	gzReader, err := gzip.NewReader(input)
	if err != nil {
		return err
	}
	defer gzReader.Close()
	reader := tar.NewReader(gzReader)
	gzWriter := gzip.NewWriter(out)
	writer := tar.NewWriter(gzWriter)
	if err := adjustTar(reader, writer, "go/", []adjustFunc{
		dropRegexpMatches(dropPatterns),
		dropUnwantedSysos(b.Target),
		fixupCrossCompile(b.Target),
		fixPermissions(),
	}); err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}
	return gzWriter.Close()
}

func (b *BuildletStep) makeEnv() []string {
	// We need GOROOT_FINAL both during the binary build and test runs. See go.dev/issue/52236.
	makeEnv := []string{"GOROOT_FINAL=" + b.BuildConfig.GorootFinal()}
	// Add extra vars from the target's configuration.
	makeEnv = append(makeEnv, b.Target.ExtraEnv...)
	return makeEnv
}

// BuildDarwinPKG builds an unsigned macOS installer
// for the given Go version and binary archive.
// It writes the result to pkg.
func (b *BuildletStep) BuildDarwinPKG(ctx *workflow.TaskContext, binaryArchive io.Reader, goVersion string, pkg io.Writer) error {
	ctx.Printf("Building inner .pkg with pkgbuild.")
	if err := b.exec(ctx, "mkdir", []string{"pkg-intermediate"}, buildlet.ExecOpts{SystemLevel: true}); err != nil {
		return err
	}
	if err := b.Buildlet.PutTar(ctx, binaryArchive, "pkg-root/usr/local"); err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, strings.NewReader("/usr/local/go/bin\n"), "pkg-root/etc/paths.d/go", 0644); err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, strings.NewReader(`#!/bin/bash

GOROOT=/usr/local/go
echo "Removing previous installation"
if [ -d $GOROOT ]; then
	rm -r $GOROOT
fi
`), "pkg-scripts/preinstall", 0755); err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, strings.NewReader(`#!/bin/bash

GOROOT=/usr/local/go
echo "Fixing permissions"
cd $GOROOT
find . -exec chmod ugo+r \{\} \;
find bin -exec chmod ugo+rx \{\} \;
find . -type d -exec chmod ugo+rx \{\} \;
chmod o-w .
`), "pkg-scripts/postinstall", 0755); err != nil {
		return err
	}
	if err := b.exec(ctx, "pkgbuild", []string{
		"--identifier=org.golang.go",
		"--version", goVersion,
		"--scripts=pkg-scripts",
		"--root=pkg-root",
		"pkg-intermediate/org.golang.go.pkg",
	}, buildlet.ExecOpts{SystemLevel: true}); err != nil {
		return err
	}

	ctx.Printf("Building outer .pkg with productbuild.")
	if err := b.exec(ctx, "mkdir", []string{"pkg-out"}, buildlet.ExecOpts{SystemLevel: true}); err != nil {
		return err
	}
	bg, err := darwinPKGBackground(b.Target.GOARCH)
	if err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, bytes.NewReader(bg), "pkg-resources/bg-light.png", 0644); err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, bytes.NewReader(bg), "pkg-resources/bg-dark.png", 0644); err != nil {
		return err
	}
	var buf bytes.Buffer
	if err := darwinDistTmpl.ExecuteTemplate(&buf, "dist.xml", darwinDistData[b.Target.GOARCH]); err != nil {
		return err
	}
	if err := b.Buildlet.Put(ctx, &buf, "pkg-distribution", 0644); err != nil {
		return err
	}
	if err := b.exec(ctx, "productbuild", []string{
		"--distribution=pkg-distribution",
		"--resources=pkg-resources",
		"--package-path=pkg-intermediate",
		"pkg-out/" + goVersion + ".pkg",
	}, buildlet.ExecOpts{SystemLevel: true}); err != nil {
		return err
	}

	return fetchFile(ctx, b.Buildlet, pkg, "pkg-out")
}

//go:embed _data/darwinpkg
var darwinPKGData embed.FS

func darwinPKGBackground(goarch string) ([]byte, error) {
	switch goarch {
	case "arm64":
		return darwinPKGData.ReadFile("_data/darwinpkg/blue-bg.png")
	case "amd64":
		return darwinPKGData.ReadFile("_data/darwinpkg/brown-bg.png")
	default:
		return nil, fmt.Errorf("no background for GOARCH %q", goarch)
	}
}

var darwinDistTmpl = template.Must(template.New("").ParseFS(darwinPKGData, "_data/darwinpkg/dist.xml"))
var darwinDistData = map[string]struct { // Map key is the target GOARCH.
	HostArchs   string // hostArchitectures option value.
	MinOS       string // Minimum required system.version.ProductVersion.
	MinOSPretty string // For example, "macOS 11".
}{
	"arm64": {
		HostArchs:   "arm64",
		MinOS:       "11.0.0",
		MinOSPretty: "macOS 11",
	},
	"amd64": {
		HostArchs:   "x86_64",
		MinOS:       "10.13.0",
		MinOSPretty: "macOS 10.13",
	},
}

// ConvertPKGToTGZ converts a macOS installer (.pkg) to a .tar.gz tarball.
func (b *BuildletStep) ConvertPKGToTGZ(ctx *workflow.TaskContext, in io.Reader, out io.Writer) error {
	if err := b.Buildlet.Put(ctx, in, "go.pkg", 0400); err != nil {
		return err
	}

	ctx.Printf("Expanding PKG installer payload.")
	if err := b.exec(ctx, "pkgutil", []string{"--expand-full", "go.pkg", "pkg-expanded"}, buildlet.ExecOpts{SystemLevel: true}); err != nil {
		return err
	}

	ctx.Printf("Compressing into a tarball.")
	input, err := b.Buildlet.GetTar(ctx, "pkg-expanded/org.golang.go.pkg/Payload/usr/local/go")
	if err != nil {
		return err
	}
	defer input.Close()

	gzReader, err := gzip.NewReader(input)
	if err != nil {
		return err
	}
	defer gzReader.Close()
	reader := tar.NewReader(gzReader)
	gzWriter := gzip.NewWriter(out)
	writer := tar.NewWriter(gzWriter)
	if err := adjustTar(reader, writer, "go/", nil); err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}
	return gzWriter.Close()
}

//go:embed releaselet/releaselet.go
var releaselet string

func (b *BuildletStep) BuildWindowsMSI(ctx *workflow.TaskContext, binaryArchive io.Reader, msi io.Writer) error {
	if err := b.Buildlet.PutTar(ctx, binaryArchive, ""); err != nil {
		return err
	}
	ctx.Printf("Pushing and running releaselet.")
	if err := b.Buildlet.Put(ctx, strings.NewReader(releaselet), "releaselet.go", 0666); err != nil {
		return err
	}
	if err := b.runGo(ctx, []string{"run", "releaselet.go"}, buildlet.ExecOpts{
		Dir: ".", // root of buildlet work directory
	}); err != nil {
		ctx.Printf("releaselet failed: %v", err)
		b.Buildlet.ListDir(ctx, ".", buildlet.ListDirOpts{Recursive: true}, func(ent buildlet.DirEntry) {
			ctx.Printf("remote: %v", ent)
		})
		return err
	}
	return fetchFile(ctx, b.Buildlet, msi, "msi")
}

// fetchFile fetches the specified directory from the given buildlet, and
// writes the first file it finds in that directory to dest.
func fetchFile(ctx *workflow.TaskContext, client buildlet.RemoteClient, dest io.Writer, dir string) error {
	ctx.Printf("Downloading file from %q.", dir)
	tgz, err := client.GetTar(context.Background(), dir)
	if err != nil {
		return err
	}
	defer tgz.Close()
	zr, err := gzip.NewReader(tgz)
	if err != nil {
		return err
	}
	tr := tar.NewReader(zr)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			return io.ErrUnexpectedEOF
		} else if err != nil {
			return err
		}
		if !h.FileInfo().IsDir() {
			break
		}
	}
	_, err = io.Copy(dest, tr)
	return err
}

func (b *BuildletStep) TestTarget(ctx *workflow.TaskContext, binaryArchive io.Reader) error {
	buildEnv := buildenv.Production
	if err := b.Buildlet.PutTar(ctx, binaryArchive, ""); err != nil {
		return err
	}
	if u := b.BuildConfig.GoBootstrapURL(buildEnv); u != "" {
		ctx.Printf("Installing go1.4 (second time, for all.bash).")
		if err := b.Buildlet.PutTarFromURL(ctx, u, go14); err != nil {
			return err
		}
	}
	ctx.Printf("Building (all.bash to ensure tests pass).")
	return b.exec(ctx, goDir+"/"+b.BuildConfig.AllScript(), b.BuildConfig.AllScriptArgs(), buildlet.ExecOpts{
		ExtraEnv: b.makeEnv(),
	})
}

func (b *BuildletStep) RunTryBot(ctx *workflow.TaskContext, sourceArchive io.Reader) (bool, error) {
	ctx.Printf("Pushing source to buildlet.")
	if err := b.Buildlet.PutTar(ctx, sourceArchive, ""); err != nil {
		return false, fmt.Errorf("failed to put generated source tarball: %v", err)
	}

	if u := b.BuildConfig.GoBootstrapURL(buildenv.Production); u != "" {
		ctx.Printf("Installing go1.4.")
		if err := b.Buildlet.PutTarFromURL(ctx, u, go14); err != nil {
			return false, err
		}
	}

	ctx.Printf("Testing")
	err := b.exec(ctx, goDir+"/"+b.BuildConfig.AllScript(), b.BuildConfig.AllScriptArgs(), buildlet.ExecOpts{})
	if err != nil {
		ctx.Printf("testing failed: %v", err)
	}
	return err == nil, nil
}

// exec runs cmd with args. Its working dir is opts.Dir, or the directory of cmd.
// Its environment is the buildlet's environment, plus a GOPATH setting, plus opts.ExtraEnv.
// If the command fails, its output is included in the returned error.
func (b *BuildletStep) exec(ctx context.Context, cmd string, args []string, opts buildlet.ExecOpts) error {
	work, err := b.Buildlet.WorkDir(ctx)
	if err != nil {
		return err
	}

	// Set up build environment. The caller's environment wins if there's a conflict.
	env := append(b.BuildConfig.Env(), "GOPATH="+work+"/gopath")
	env = append(env, opts.ExtraEnv...)
	opts.Output = b.LogWriter
	opts.ExtraEnv = env
	opts.Args = args
	remoteErr, execErr := b.Buildlet.Exec(ctx, cmd, opts)
	if execErr != nil {
		return execErr
	}
	if remoteErr != nil {
		return fmt.Errorf("Command %v %s failed: %v", cmd, args, remoteErr)
	}

	return nil
}

func (b *BuildletStep) runGo(ctx context.Context, args []string, execOpts buildlet.ExecOpts) error {
	goCmd := goDir + "/bin/go"
	if b.Target.GOOS == "windows" {
		goCmd += ".exe"
	}
	execOpts.Args = args
	return b.exec(ctx, goCmd, args, execOpts)
}

func ConvertTGZToZIP(r io.Reader, w io.Writer) error {
	zr, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	tr := tar.NewReader(zr)

	zw := zip.NewWriter(w)
	for {
		th, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		fi := th.FileInfo()
		zh, err := zip.FileInfoHeader(fi)
		if err != nil {
			return err
		}
		zh.Name = th.Name // for the full path
		switch strings.ToLower(path.Ext(zh.Name)) {
		case ".jpg", ".jpeg", ".png", ".gif":
			// Don't re-compress already compressed files.
			zh.Method = zip.Store
		default:
			zh.Method = zip.Deflate
		}
		if fi.IsDir() {
			zh.Method = zip.Store
		}
		w, err := zw.CreateHeader(zh)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			continue
		}
		if _, err := io.Copy(w, tr); err != nil {
			return err
		}
	}
	return zw.Close()
}