internal/task/cloudbuild.go

// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package task

import (
	"context"
	"fmt"
	"io/fs"
	"math/rand"
	"strings"

	cloudbuild "cloud.google.com/go/cloudbuild/apiv1/v2"
	"cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb"
	"cloud.google.com/go/storage"
	"golang.org/x/build/internal/gcsfs"
)

type CloudBuildClient interface {
	// RunBuildTrigger runs an existing trigger in project with the given
	// substitutions.
	RunBuildTrigger(ctx context.Context, project, trigger string, substitutions map[string]string) (CloudBuild, error)
	// RunScript runs the given script under bash -eux -o pipefail in
	// ScriptProject. Outputs are collected into the build's ResultURL,
	// readable with ResultFS. The script will have the latest version of Go
	// and some version of gsutil on $PATH.
	// If gerritProject is specified, the script will run in the root of a
	// checkout of the tip version of that repository.
	RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error)
	// Completed reports whether a build has finished, returning an error if
	// it's failed. It's suitable for use with AwaitCondition.
	Completed(ctx context.Context, build CloudBuild) (detail string, completed bool, _ error)
	// ResultFS returns an FS that contains the results of the given build.
	ResultFS(ctx context.Context, build CloudBuild) (fs.FS, error)
}

type RealCloudBuildClient struct {
	BuildClient   *cloudbuild.Client
	StorageClient *storage.Client
	ScriptProject string
	ScriptAccount string
	ScratchURL    string
}

// CloudBuild represents a Cloud Build that can be queried with the status
// methods on CloudBuildClient.
type CloudBuild struct {
	Project, ID string
	ResultURL   string
}

func (c *RealCloudBuildClient) RunBuildTrigger(ctx context.Context, project, trigger string, substitutions map[string]string) (CloudBuild, error) {
	op, err := c.BuildClient.RunBuildTrigger(ctx, &cloudbuildpb.RunBuildTriggerRequest{
		ProjectId: project,
		TriggerId: trigger,
		Source: &cloudbuildpb.RepoSource{
			Substitutions: substitutions,
		},
	})
	if err != nil {
		return CloudBuild{}, err
	}
	if _, err = op.Poll(ctx); err != nil {
		return CloudBuild{}, err
	}
	meta, err := op.Metadata()
	if err != nil {
		return CloudBuild{}, err
	}
	return CloudBuild{Project: project, ID: meta.Build.Id}, nil
}

func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) {
	const downloadGoScript = `#!/usr/bin/env bash
set -eux
archive=$(wget -qO - 'https://go.dev/dl/?mode=json' | grep -Eo 'go.*linux-amd64.tar.gz' | head -n 1)
wget -qO - https://go.dev/dl/${archive} | tar -xz
mv go /workspace/released_go
`

	const scriptPrefix = `#!/usr/bin/env bash
set -eux
set -o pipefail
export PATH=/workspace/released_go/bin:$PATH
`

	// Cloud build loses directory structure when it saves artifacts, which is
	// a problem since (e.g.) we have multiple files named go.mod in the
	// tagging tasks. It's not very complicated, so reimplement it ourselves.
	resultURL := fmt.Sprintf("%v/script-build-%v", c.ScratchURL, rand.Int63())
	var saveOutputsScript strings.Builder
	saveOutputsScript.WriteString(scriptPrefix)
	for _, out := range outputs {
		saveOutputsScript.WriteString(fmt.Sprintf("gsutil cp %q %q\n", out, resultURL+"/"+strings.TrimPrefix(out, "./")))
	}

	var steps []*cloudbuildpb.BuildStep
	var dir string
	if gerritProject != "" {
		steps = append(steps, &cloudbuildpb.BuildStep{
			Name: "gcr.io/cloud-builders/git",
			Args: []string{"clone", "https://go.googlesource.com/" + gerritProject, "checkout"},
		})
		dir = "checkout"
	}

	build := &cloudbuildpb.Build{
		Steps: append(steps,
			&cloudbuildpb.BuildStep{
				Name:   "bash",
				Script: downloadGoScript,
			},
			&cloudbuildpb.BuildStep{
				Name:   "gcr.io/cloud-builders/gsutil",
				Script: scriptPrefix + script,
				Dir:    dir,
			},
			&cloudbuildpb.BuildStep{
				Name:   "gcr.io/cloud-builders/gsutil",
				Script: saveOutputsScript.String(),
				Dir:    dir,
			},
		),
		Options: &cloudbuildpb.BuildOptions{
			MachineType: cloudbuildpb.BuildOptions_E2_HIGHCPU_8,
			Logging:     cloudbuildpb.BuildOptions_CLOUD_LOGGING_ONLY,
		},
		ServiceAccount: c.ScriptAccount,
	}
	op, err := c.BuildClient.CreateBuild(ctx, &cloudbuildpb.CreateBuildRequest{
		ProjectId: c.ScriptProject,
		Build:     build,
	})
	if err != nil {
		return CloudBuild{}, fmt.Errorf("creating build: %w", err)
	}
	if _, err = op.Poll(ctx); err != nil {
		return CloudBuild{}, fmt.Errorf("polling: %w", err)
	}
	meta, err := op.Metadata()
	if err != nil {
		return CloudBuild{}, fmt.Errorf("reading metadata: %w", err)
	}
	return CloudBuild{Project: c.ScriptProject, ID: meta.Build.Id, ResultURL: resultURL}, nil

}

func (c *RealCloudBuildClient) Completed(ctx context.Context, build CloudBuild) (string, bool, error) {
	b, err := c.BuildClient.GetBuild(ctx, &cloudbuildpb.GetBuildRequest{
		ProjectId: build.Project,
		Id:        build.ID,
	})
	if err != nil {
		return "", false, err
	}
	if b.FinishTime == nil {
		return "", false, nil
	}
	if b.Status != cloudbuildpb.Build_SUCCESS {
		return "", false, fmt.Errorf("build %q failed, see %v: %v", build.ID, build.ResultURL, b.FailureInfo)
	}
	return b.StatusDetail, true, nil
}

func (c *RealCloudBuildClient) ResultFS(ctx context.Context, build CloudBuild) (fs.FS, error) {
	return gcsfs.FromURL(ctx, c.StorageClient, build.ResultURL)
}