blob: fd995832f8e982c36c1f16b2ce622bac5d175d9e [file] [log] [blame]
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package task
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"io/fs"
"math/rand"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
pb "go.chromium.org/luci/buildbucket/proto"
"golang.org/x/build/buildlet"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/gcsfs"
"golang.org/x/build/internal/relui/sign"
"golang.org/x/build/internal/untar"
wf "golang.org/x/build/internal/workflow"
"golang.org/x/exp/slices"
"google.golang.org/protobuf/types/known/structpb"
)
// ServeTarball serves files as a .tar.gz to w, only if path contains pathMatch.
func ServeTarball(pathMatch string, files map[string]string, w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.URL.Path, pathMatch) {
w.WriteHeader(http.StatusNotFound)
return
}
tgz, err := mapToTgz(files)
if err != nil {
panic(err)
}
if _, err := w.Write(tgz); err != nil {
panic(err)
}
}
func mapToTgz(files map[string]string) ([]byte, error) {
w := &bytes.Buffer{}
gzw := gzip.NewWriter(w)
tw := tar.NewWriter(gzw)
for name, contents := range files {
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: name,
Size: int64(len(contents)),
Mode: 0777,
ModTime: time.Now(),
AccessTime: time.Now(),
ChangeTime: time.Now(),
}); err != nil {
return nil, err
}
if _, err := tw.Write([]byte(contents)); err != nil {
return nil, err
}
}
if err := tw.Close(); err != nil {
return nil, err
}
if err := gzw.Close(); err != nil {
return nil, err
}
return w.Bytes(), nil
}
// NewFakeBuildlets creates a set of fake buildlets.
// httpServer is the base URL of form http://host with no trailing slash
// where PutTarFromURL downloads remote URLs from.
// sysCmds optionally allows overriding the named system commands
// during testing with the given executable content.
func NewFakeBuildlets(t *testing.T, httpServer string, sysCmds map[string]string) *FakeBuildlets {
var sys map[string]string
if len(sysCmds) != 0 {
sys = make(map[string]string)
sysDir := t.TempDir()
for name, content := range sysCmds {
if err := os.WriteFile(filepath.Join(sysDir, name), []byte(content), 0700); err != nil {
t.Fatal(err)
}
sys[name] = filepath.Join(sysDir, name)
}
}
return &FakeBuildlets{
t: t,
dir: t.TempDir(),
sys: sys,
httpURL: httpServer,
logs: map[string][]*[]string{},
}
}
type FakeBuildlets struct {
t *testing.T
dir string
sys map[string]string // System command name → absolute path.
httpURL string
mu sync.Mutex
nextID int
logs map[string][]*[]string
}
func (b *FakeBuildlets) CreateBuildlet(_ context.Context, kind string) (buildlet.RemoteClient, error) {
b.mu.Lock()
buildletDir := filepath.Join(b.dir, kind, fmt.Sprint(b.nextID), "work")
if err := os.MkdirAll(buildletDir, 0700); err != nil {
return nil, err
}
tempDir := filepath.Join(b.dir, kind, fmt.Sprint(b.nextID), "tmp")
if err := os.MkdirAll(tempDir, 0700); err != nil {
return nil, err
}
logs := &[]string{}
b.nextID++
b.logs[kind] = append(b.logs[kind], logs)
b.mu.Unlock()
logf := func(format string, args ...interface{}) {
line := fmt.Sprintf(format, args...)
line = strings.ReplaceAll(line, buildletDir, "$WORK")
*logs = append(*logs, line)
}
logf("--- create buildlet ---")
return &fakeBuildlet{
t: b.t,
kind: kind,
workDir: buildletDir,
tempDir: tempDir,
sys: b.sys,
httpURL: b.httpURL,
logf: logf,
}, nil
}
func (b *FakeBuildlets) DumpLogs() {
for name, logs := range b.logs {
b.t.Logf("%v buildlets:", name)
for _, group := range logs {
for _, line := range *group {
b.t.Log(line)
}
}
}
}
type fakeBuildlet struct {
buildlet.Client
t *testing.T
kind string
workDir string
tempDir string
sys map[string]string // System command name → absolute path.
httpURL string
logf func(string, ...interface{})
closed bool
}
func (b *fakeBuildlet) Close() error {
if !b.closed {
b.logf("--- destroy buildlet ---")
b.closed = true
}
return nil
}
func (b *fakeBuildlet) Exec(ctx context.Context, cmd string, opts buildlet.ExecOpts) (remoteErr error, execErr error) {
// TODO: add support for opts.Path. Previously, setting opts.Path would cause
// an error here, but that caused unnecessary failures in tests that use mock
// execution.
if opts.OnStartExec != nil {
return nil, fmt.Errorf("opts.OnStartExec option is set, but fakeBuildlet doesn't support it")
}
b.logf("exec %v %v\n\twd %q env %v", cmd, opts.Args, opts.Dir, opts.ExtraEnv)
if absPath, ok := b.sys[cmd]; ok && opts.SystemLevel {
cmd = absPath
} else if !strings.HasPrefix(cmd, "/") && !opts.SystemLevel {
cmd = filepath.Join(b.workDir, cmd)
}
retry:
c := exec.CommandContext(ctx, cmd, opts.Args...)
c.Env = append(os.Environ(), opts.ExtraEnv...)
c.Env = append(c.Env, "TEMP="+b.tempDir, "TMP="+b.tempDir, "TEMPDIR="+b.tempDir, "TMPDIR="+b.tempDir)
buf := &bytes.Buffer{}
var w io.Writer = buf
if opts.Output != nil {
w = io.MultiWriter(w, opts.Output)
}
c.Stdout = w
c.Stderr = w
if opts.Dir == "" && opts.SystemLevel {
c.Dir = b.workDir
} else if opts.Dir == "" && !opts.SystemLevel {
c.Dir = filepath.Dir(cmd)
} else {
c.Dir = filepath.Join(b.workDir, opts.Dir)
}
err := c.Run()
// Work around Unix foolishness. See go.dev/issue/22315.
if err != nil && strings.Contains(err.Error(), "text file busy") {
time.Sleep(100 * time.Millisecond)
goto retry
}
if err != nil {
return nil, fmt.Errorf("command %v %v failed: %v output: %q", cmd, opts.Args, err, buf.String())
}
return nil, nil
}
func (b *fakeBuildlet) GetTar(ctx context.Context, dir string) (io.ReadCloser, error) {
b.logf("get tar of %q", dir)
buf := &bytes.Buffer{}
zw := gzip.NewWriter(buf)
tw := tar.NewWriter(zw)
base := filepath.Join(b.workDir, filepath.FromSlash(dir))
// Copied pretty much wholesale from buildlet.go.
err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
rel := strings.TrimPrefix(filepath.ToSlash(strings.TrimPrefix(path, base)), "/")
th, err := tar.FileInfoHeader(fi, path)
if err != nil {
return err
}
th.Name = rel
if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
th.Name += "/"
}
if th.Name == "/" {
return nil
}
if err := tw.WriteHeader(th); err != nil {
return err
}
if fi.Mode().IsRegular() {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(tw, f); err != nil {
return err
}
}
return nil
})
if err != nil {
return nil, err
}
if err := tw.Close(); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
return io.NopCloser(buf), nil
}
func (b *fakeBuildlet) ListDir(ctx context.Context, dir string, opts buildlet.ListDirOpts, fn func(buildlet.DirEntry)) error {
// We call this when something goes wrong, so we need it to "succeed".
// It's not worth implementing; return some nonsense.
fn(buildlet.DirEntry{
Line: "ListDir is silently unimplemented, sorry",
})
return nil
}
func (b *fakeBuildlet) Put(ctx context.Context, r io.Reader, path string, mode os.FileMode) error {
b.logf("write file %q with mode %0o", path, mode)
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.workDir, path)), 0755); err != nil {
return err
}
f, err := os.OpenFile(filepath.Join(b.workDir, path), os.O_CREATE|os.O_RDWR, mode)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(f, r); err != nil {
return err
}
return f.Close()
}
func (b *fakeBuildlet) PutTar(ctx context.Context, r io.Reader, dir string) error {
b.logf("put tar to %q", dir)
return untar.Untar(r, filepath.Join(b.workDir, dir))
}
func (b *fakeBuildlet) PutTarFromURL(ctx context.Context, tarURL string, dir string) error {
url, err := url.Parse(tarURL)
if err != nil {
return err
}
rewritten := url.String()
if !strings.Contains(url.Host, "localhost") && !strings.Contains(url.Host, "127.0.0.1") {
rewritten = b.httpURL + url.Path
}
b.logf("put tar from %v (rewritten to %v) to %q", tarURL, rewritten, dir)
resp, err := http.Get(rewritten)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status for %q: %v", tarURL, resp.Status)
}
defer resp.Body.Close()
return untar.Untar(resp.Body, filepath.Join(b.workDir, dir))
}
func (b *fakeBuildlet) WorkDir(ctx context.Context) (string, error) {
return b.workDir, nil
}
func NewFakeGerrit(t *testing.T, repos ...*FakeRepo) *FakeGerrit {
result := &FakeGerrit{
repos: map[string]*FakeRepo{},
}
server := httptest.NewServer(http.HandlerFunc(result.serveHTTP))
result.serverURL = server.URL
t.Cleanup(server.Close)
for _, r := range repos {
result.repos[r.name] = r
}
return result
}
type FakeGerrit struct {
repos map[string]*FakeRepo
serverURL string
}
type FakeRepo struct {
t *testing.T
name string
dir *GitDir
}
func NewFakeRepo(t *testing.T, name string) *FakeRepo {
if _, err := exec.LookPath("git"); errors.Is(err, exec.ErrNotFound) {
t.Skip("test requires git")
}
r := &FakeRepo{
t: t,
name: name,
dir: &GitDir{&Git{}, t.TempDir()},
}
t.Cleanup(func() { r.dir.Close() })
r.runGit("init")
r.runGit("commit", "--allow-empty", "--allow-empty-message", "-m", "")
return r
}
// TODO(rfindley): probably every method on FakeRepo should invoke
// repo.t.Helper(), otherwise it's impossible to see where the test failed.
func (repo *FakeRepo) runGit(args ...string) []byte {
repo.t.Helper()
configArgs := []string{
"-c", "init.defaultBranch=master",
"-c", "user.email=relui@example.com",
"-c", "user.name=relui",
}
out, err := repo.dir.RunCommand(context.Background(), append(configArgs, args...)...)
if err != nil {
repo.t.Fatalf("runGit(%v) failed: %v; output:\n%s", args, err, out)
}
return out
}
func (repo *FakeRepo) Commit(contents map[string]string) string {
return repo.CommitOnBranch("master", contents)
}
func (repo *FakeRepo) CommitOnBranch(branch string, contents map[string]string) string {
repo.runGit("switch", branch)
for k, v := range contents {
full := filepath.Join(repo.dir.dir, k)
if err := os.MkdirAll(filepath.Dir(full), 0777); err != nil {
repo.t.Fatal(err)
}
if err := os.WriteFile(full, []byte(v), 0777); err != nil {
repo.t.Fatal(err)
}
}
repo.runGit("add", ".")
repo.runGit("commit", "--allow-empty-message", "-m", "")
return strings.TrimSpace(string(repo.runGit("rev-parse", "HEAD")))
}
func (repo *FakeRepo) History() []string {
return strings.Split(string(repo.runGit("log", "--format=%H")), "\n")
}
func (repo *FakeRepo) Tag(tag, commit string) {
repo.runGit("tag", tag, commit)
}
func (repo *FakeRepo) Branch(branch, commit string) {
repo.runGit("branch", branch, commit)
}
func (repo *FakeRepo) ReadFile(commit, file string) ([]byte, error) {
b, err := repo.dir.RunCommand(context.Background(), "show", commit+":"+file)
if err != nil && strings.Contains(err.Error(), " does not exist ") {
err = errors.Join(gerrit.ErrResourceNotExist, err)
}
return b, err
}
var _ GerritClient = (*FakeGerrit)(nil)
func (g *FakeGerrit) GitilesURL() string {
return g.serverURL
}
func (g *FakeGerrit) ListProjects(ctx context.Context) ([]string, error) {
var names []string
for k := range g.repos {
names = append(names, k)
}
return names, nil
}
func (g *FakeGerrit) repo(name string) (*FakeRepo, error) {
if r, ok := g.repos[name]; ok {
return r, nil
} else {
return nil, fmt.Errorf("no such repo %v: %w", name, gerrit.ErrResourceNotExist)
}
}
func (g *FakeGerrit) ReadBranchHead(ctx context.Context, project, branch string) (string, error) {
repo, err := g.repo(project)
if err != nil {
return "", err
}
// TODO: If the branch doesn't exist, return an error matching gerrit.ErrResourceNotExist.
out, err := repo.dir.RunCommand(ctx, "rev-parse", "refs/heads/"+branch)
return strings.TrimSpace(string(out)), err
}
func (g *FakeGerrit) ReadFile(ctx context.Context, project string, commit string, file string) ([]byte, error) {
repo, err := g.repo(project)
if err != nil {
return nil, err
}
return repo.ReadFile(commit, file)
}
func (g *FakeGerrit) ListTags(ctx context.Context, project string) ([]string, error) {
repo, err := g.repo(project)
if err != nil {
return nil, err
}
out, err := repo.dir.RunCommand(ctx, "tag", "-l")
if err != nil {
return nil, err
}
if len(out) == 0 {
return nil, nil // No tags.
}
return strings.Split(strings.TrimSpace(string(out)), "\n"), nil
}
func (g *FakeGerrit) GetTag(ctx context.Context, project string, tag string) (gerrit.TagInfo, error) {
repo, err := g.repo(project)
if err != nil {
return gerrit.TagInfo{}, err
}
out, err := repo.dir.RunCommand(ctx, "rev-parse", "refs/tags/"+tag)
return gerrit.TagInfo{Revision: strings.TrimSpace(string(out))}, err
}
func (g *FakeGerrit) CreateAutoSubmitChange(_ *wf.TaskContext, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {
repo, err := g.repo(input.Project)
if err != nil {
return "", err
}
commit := repo.CommitOnBranch(input.Branch, contents)
return "cl_" + commit, nil
}
func (g *FakeGerrit) Submitted(ctx context.Context, changeID, baseCommit string) (string, bool, error) {
return strings.TrimPrefix(changeID, "cl_"), true, nil
}
func (g *FakeGerrit) Tag(ctx context.Context, project, tag, commit string) error {
repo, err := g.repo(project)
if err != nil {
return err
}
repo.Tag(tag, commit)
return nil
}
func (g *FakeGerrit) GetCommitsInRefs(ctx context.Context, project string, commits, refs []string) (map[string][]string, error) {
repo, err := g.repo(project)
if err != nil {
return nil, err
}
refSet := map[string]bool{}
for _, ref := range refs {
refSet[ref] = true
}
result := map[string][]string{}
for _, commit := range commits {
out, err := repo.dir.RunCommand(ctx, "branch", "--format=%(refname)", "--contains="+commit)
if err != nil {
return nil, err
}
for _, branch := range strings.Split(strings.TrimSpace(string(out)), "\n") {
branch := strings.TrimSpace(branch)
if refSet[branch] {
result[commit] = append(result[commit], branch)
}
}
}
return result, nil
}
func (g *FakeGerrit) GerritURL() string {
return g.serverURL
}
func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
if len(parts) != 4 {
w.WriteHeader(http.StatusNotFound)
return
}
repo, err := g.repo(parts[1])
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
rev := strings.TrimSuffix(parts[3], ".tar.gz")
archive, err := repo.dir.RunCommand(r.Context(), "archive", "--format=tgz", rev)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
http.ServeContent(w, r, parts[3], time.Now(), bytes.NewReader(archive))
}
func (*FakeGerrit) QueryChanges(_ context.Context, query string) ([]*gerrit.ChangeInfo, error) {
return nil, nil
}
func (*FakeGerrit) SetHashtags(_ context.Context, changeID string, _ gerrit.HashtagsInput) error {
return fmt.Errorf("pretend that SetHashtags failed")
}
// NewFakeSignService returns a fake signing service that can sign PKGs, MSIs,
// and generate GPG signatures. MSIs are "signed" by adding a suffix to them.
// PKGs must actually be tarballs with a prefix of "I'm a PKG!\n". Any files
// they contain that look like binaries will be "signed".
func NewFakeSignService(t *testing.T, outputDir string) *FakeSignService {
return &FakeSignService{
t: t,
outputDir: outputDir,
completedJobs: map[string][]string{},
}
}
type FakeSignService struct {
t *testing.T
outputDir string
mu sync.Mutex
completedJobs map[string][]string // Job ID → output objectURIs.
}
func (s *FakeSignService) SignArtifact(_ context.Context, bt sign.BuildType, in []string) (jobID string, _ error) {
s.t.Logf("fakeSignService: doing %s signing of %q", bt, in)
jobID = uuid.NewString()
var out []string
switch bt {
case sign.BuildMacOS:
if len(in) != 1 {
return "", fmt.Errorf("got %d inputs, want 1", len(in))
}
out = []string{s.fakeSignPKG(jobID, in[0], fmt.Sprintf("-signed <%s>", bt))}
case sign.BuildWindows:
if len(in) != 1 {
return "", fmt.Errorf("got %d inputs, want 1", len(in))
}
out = []string{s.fakeSignFile(jobID, in[0], fmt.Sprintf("-signed <%s>", bt))}
case sign.BuildGPG:
if len(in) == 0 {
return "", fmt.Errorf("got 0 inputs, want 1 or more")
}
for _, f := range in {
out = append(out, s.fakeGPGFile(jobID, f))
}
default:
return "", fmt.Errorf("SignArtifact: not implemented for %v", bt)
}
s.mu.Lock()
s.completedJobs[jobID] = out
s.mu.Unlock()
return jobID, nil
}
func (s *FakeSignService) ArtifactSigningStatus(_ context.Context, jobID string) (_ sign.Status, desc string, out []string, _ error) {
s.mu.Lock()
out, ok := s.completedJobs[jobID]
s.mu.Unlock()
if !ok {
return sign.StatusNotFound, fmt.Sprintf("job %q not found", jobID), nil, nil
}
return sign.StatusCompleted, "", out, nil
}
func (s *FakeSignService) CancelSigning(_ context.Context, jobID string) error {
s.t.Errorf("CancelSigning was called unexpectedly")
return fmt.Errorf("intentional fake error")
}
func (s *FakeSignService) fakeSignPKG(jobID, f, msg string) string {
b, err := os.ReadFile(strings.TrimPrefix(f, "file://"))
if err != nil {
panic(fmt.Errorf("fakeSignPKG: os.ReadFile: %v", err))
}
b = bytes.TrimPrefix(b, []byte("I'm a PKG!\n"))
files, err := tgzToMap(bytes.NewReader(b))
if err != nil {
panic(fmt.Errorf("fakeSignPKG: tgzToMap: %v", err))
}
for fn, contents := range files {
if !strings.Contains(fn, "go/bin") && !strings.Contains(fn, "go/pkg/tool") {
continue
}
files[fn] = contents + msg
}
b, err = mapToTgz(files)
if err != nil {
panic(fmt.Errorf("fakeSignPKG: mapToTgz: %v", err))
}
b = append([]byte("I'm a PKG! "+msg+"\n"), b...)
return s.writeOutput(jobID, path.Base(f), b)
}
func (s *FakeSignService) writeOutput(jobID, base string, contents []byte) string {
path := path.Join(s.outputDir, jobID, base)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
panic(fmt.Errorf("fake signing service: os.MkdirAll: %v", err))
}
if err := os.WriteFile(path, contents, 0600); err != nil {
panic(fmt.Errorf("fake signing service: os.WriteFile: %v", err))
}
return "file://" + path
}
func tgzToMap(r io.Reader) (map[string]string, error) {
gzr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
defer gzr.Close()
result := map[string]string{}
tr := tar.NewReader(gzr)
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if h.Typeflag != tar.TypeReg {
continue
}
b, err := io.ReadAll(tr)
if err != nil {
return nil, err
}
result[h.Name] = string(b)
}
return result, nil
}
func (s *FakeSignService) fakeSignFile(jobID, f, msg string) string {
b, err := os.ReadFile(strings.TrimPrefix(f, "file://"))
if err != nil {
panic(fmt.Errorf("fakeSignFile: os.ReadFile: %v", err))
}
b = append(b, []byte(msg)...)
return s.writeOutput(jobID, path.Base(f), b)
}
func (s *FakeSignService) fakeGPGFile(jobID, f string) string {
b, err := os.ReadFile(strings.TrimPrefix(f, "file://"))
if err != nil {
panic(fmt.Errorf("fakeGPGFile: os.ReadFile: %v", err))
}
gpg := fmt.Sprintf("I'm a GPG signature for %x!", sha256.Sum256(b))
return s.writeOutput(jobID, path.Base(f)+".asc", []byte(gpg))
}
var _ CloudBuildClient = (*FakeCloudBuild)(nil)
const fakeGsutil = `
#!/bin/bash -eux
case "$1" in
"cp")
in=$2
out=$3
if [[ $in == '-' ]]; then
in=/dev/stdin
fi
if [[ $out == '-' ]]; then
out=/dev/stdout
fi
cp "${in#file://}" "${out#file://}"
;;
"cat")
cat "${2#file://}"
;;
*)
echo unexpected command $@ >&2
exit 1
;;
esac
`
func NewFakeCloudBuild(t *testing.T, gerrit *FakeGerrit, project string, allowedTriggers map[string]map[string]string, fakeGo string) *FakeCloudBuild {
toolDir := t.TempDir()
if err := os.WriteFile(filepath.Join(toolDir, "go"), []byte(fakeGo), 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(toolDir, "gsutil"), []byte(fakeGsutil), 0777); err != nil {
t.Fatal(err)
}
return &FakeCloudBuild{
t: t,
gerrit: gerrit,
project: project,
allowedTriggers: allowedTriggers,
toolDir: toolDir,
results: map[string]error{},
}
}
type FakeCloudBuild struct {
t *testing.T
gerrit *FakeGerrit
project string
allowedTriggers map[string]map[string]string
toolDir string
mu sync.Mutex
results map[string]error
}
func (cb *FakeCloudBuild) RunBuildTrigger(ctx context.Context, project string, trigger string, substitutions map[string]string) (CloudBuild, error) {
if project != cb.project {
return CloudBuild{}, fmt.Errorf("unexpected project %v, want %v", project, cb.project)
}
if allowedSubs, ok := cb.allowedTriggers[trigger]; !ok || !reflect.DeepEqual(allowedSubs, substitutions) {
return CloudBuild{}, fmt.Errorf("unexpected trigger %v: got params %#v, want %#v", trigger, substitutions, allowedSubs)
}
id := fmt.Sprintf("build-%v", rand.Int63())
cb.mu.Lock()
cb.results[id] = nil
cb.mu.Unlock()
return CloudBuild{Project: project, ID: id}, nil
}
func (cb *FakeCloudBuild) Completed(ctx context.Context, build CloudBuild) (string, bool, error) {
if build.Project != cb.project {
return "", false, fmt.Errorf("unexpected build project: got %q, want %q", build.Project, cb.project)
}
cb.mu.Lock()
result, ok := cb.results[build.ID]
cb.mu.Unlock()
if !ok {
return "", false, fmt.Errorf("unknown build ID %q", build.ID)
}
return "here's some build detail", true, result
}
func (c *FakeCloudBuild) ResultFS(ctx context.Context, build CloudBuild) (fs.FS, error) {
return gcsfs.FromURL(ctx, nil, build.ResultURL)
}
func (cb *FakeCloudBuild) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) {
var wd string
if gerritProject != "" {
repo, err := cb.gerrit.repo(gerritProject)
if err != nil {
return CloudBuild{}, err
}
dir, err := (&Git{}).Clone(ctx, repo.dir.dir)
if err != nil {
return CloudBuild{}, err
}
defer dir.Close()
wd = dir.dir
} else {
wd = cb.t.TempDir()
}
tempDir := cb.t.TempDir()
cmd := exec.Command("bash", "-eux")
cmd.Stdin = strings.NewReader(script)
cmd.Dir = wd
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "TEMP="+tempDir, "TMP="+tempDir, "TEMPDIR="+tempDir, "TMPDIR="+tempDir)
cmd.Env = append(cmd.Env, "PATH="+cb.toolDir+":/bin:/usr/bin")
buf := &bytes.Buffer{}
cmd.Stdout = buf
cmd.Stderr = buf
runErr := cmd.Run()
if runErr != nil {
runErr = fmt.Errorf("script failed: %v output:\n%s", runErr, buf.String())
}
id := fmt.Sprintf("build-%v", rand.Int63())
resultDir := cb.t.TempDir()
if runErr == nil {
for _, out := range outputs {
target := filepath.Join(resultDir, out)
os.MkdirAll(filepath.Dir(target), 0777)
if err := os.Rename(filepath.Join(wd, out), target); err != nil {
runErr = fmt.Errorf("collecting outputs: %v", err)
break
}
}
}
cb.mu.Lock()
cb.results[id] = runErr
cb.mu.Unlock()
return CloudBuild{Project: cb.project, ID: id, ResultURL: "file://" + resultDir}, nil
}
type FakeSwarmingClient struct {
t *testing.T
toolDir string
mu sync.Mutex
results map[string]error
}
func NewFakeSwarmingClient(t *testing.T, fakeGo string) *FakeSwarmingClient {
toolDir := t.TempDir()
if err := os.WriteFile(filepath.Join(toolDir, "go"), []byte(fakeGo), 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(toolDir, "gsutil"), []byte(fakeGsutil), 0777); err != nil {
t.Fatal(err)
}
return &FakeSwarmingClient{
t: t,
toolDir: toolDir,
results: map[string]error{},
}
}
var _ SwarmingClient = (*FakeSwarmingClient)(nil)
func (c *FakeSwarmingClient) RunTask(ctx context.Context, dims map[string]string, script string, env map[string]string) (string, error) {
tempDir := c.t.TempDir()
cmd := exec.Command("bash", "-eux")
cmd.Stdin = strings.NewReader("set -o pipefail\n" + script)
cmd.Dir = c.t.TempDir()
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "TEMP="+tempDir, "TMP="+tempDir, "TEMPDIR="+tempDir, "TMPDIR="+tempDir)
cmd.Env = append(cmd.Env, "PATH="+c.toolDir+":/bin:/usr/bin:.") // Note: . is on PATH to help with Windows compatibility
for k, v := range env {
cmd.Env = append(cmd.Env, k+"="+v)
}
buf := &bytes.Buffer{}
cmd.Stdout = buf
cmd.Stderr = buf
runErr := cmd.Run()
if runErr != nil {
runErr = fmt.Errorf("script failed: %v output:\n%s", runErr, buf.String())
}
id := fmt.Sprintf("build-%v", rand.Int63())
c.mu.Lock()
c.results[id] = runErr
c.mu.Unlock()
return id, nil
}
func (c *FakeSwarmingClient) Completed(ctx context.Context, id string) (string, bool, error) {
c.mu.Lock()
result, ok := c.results[id]
c.mu.Unlock()
if !ok {
return "", false, fmt.Errorf("unknown task ID %q", id)
}
return "here's some build detail", true, result
}
func NewFakeBuildBucketClient(major int, url, bucket string, projects []string) *FakeBuildBucketClient {
return &FakeBuildBucketClient{
Bucket: bucket,
major: major,
GerritURL: url,
Projects: projects,
results: map[int64]error{},
}
}
type FakeBuildBucketClient struct {
Bucket string
FailBuilds []string
MissingBuilds []string
major int
GerritURL, Branch string
Projects []string
mu sync.Mutex
results map[int64]error
}
var _ BuildBucketClient = (*FakeBuildBucketClient)(nil)
func (c *FakeBuildBucketClient) ListBuilders(ctx context.Context, bucket string) (map[string]*pb.BuilderConfig, error) {
if bucket != c.Bucket {
return nil, fmt.Errorf("unexpected bucket %q", bucket)
}
res := map[string]*pb.BuilderConfig{}
for _, proj := range c.Projects {
prefix := ""
if proj != "go" {
prefix = "x_" + proj + "-"
}
for _, v := range []string{"gotip", fmt.Sprintf("go1.%v", c.major)} {
for _, b := range []string{"linux-amd64", "linux-amd64-longtest", "darwin-amd64_13"} {
parts := strings.FieldsFunc(b, func(r rune) bool { return r == '-' || r == '_' })
res[prefix+v+"-"+b] = &pb.BuilderConfig{
Properties: fmt.Sprintf(`{"project":%q, "is_google":true, "target":{"goos":%q, "goarch":%q}}`, proj, parts[0], parts[1]),
}
}
}
}
return res, nil
}
func (c *FakeBuildBucketClient) RunBuild(ctx context.Context, bucket string, builder string, commit *pb.GitilesCommit, properties map[string]*structpb.Value) (int64, error) {
if bucket != c.Bucket {
return 0, fmt.Errorf("unexpected bucket %q", bucket)
}
match := regexp.MustCompile(`.*://(.+)`).FindStringSubmatch(c.GerritURL)
if commit.Host != match[1] || !slices.Contains(c.Projects, commit.Project) {
return 0, fmt.Errorf("unexpected host or project: got %q, %q want %q, %q", commit.Host, commit.Project, match[1], c.Projects)
}
// It would be nice to validate the commit hash and branch, but it's
// tricky to get the right value because it depends on the release type.
// At least validate the commit is a commit.
if len(commit.Id) != 40 {
return 0, fmt.Errorf("malformed Git commit hash %q", commit.Id)
}
var runErr error
for _, failBuild := range c.FailBuilds {
if strings.Contains(builder, failBuild) {
runErr = fmt.Errorf("run of %q is specified to fail", builder)
}
}
id := rand.Int63()
c.mu.Lock()
c.results[id] = runErr
c.mu.Unlock()
return id, nil
}
func (c *FakeBuildBucketClient) Completed(ctx context.Context, id int64) (string, bool, error) {
c.mu.Lock()
result, ok := c.results[id]
c.mu.Unlock()
if !ok {
return "", false, fmt.Errorf("unknown task ID %q", id)
}
return "here's some build detail", true, result
}
func (c *FakeBuildBucketClient) SearchBuilds(ctx context.Context, pred *pb.BuildPredicate) ([]int64, error) {
if slices.Contains(c.MissingBuilds, pred.GetBuilder().GetBuilder()) {
return nil, nil
}
return []int64{rand.Int63()}, nil
}