internal/task: integration test for x repo tagging
Add an integration test for the x repo tagging workflow. Most of the
work is test infrastructure. Move the fake buildlet types into the task
package, and create a new fake Gerrit implementation that tracks tags
and commits. Also use it in the older release integration tests.
The coverage of the x repo tagging process itself is a bit thin but this
is a good starting point.
For golang/go#48523.
Change-Id: Iabf9391b56f6becadd401b8ae37a9e4b8cc5d24f
Reviewed-on: https://go-review.googlesource.com/c/build/+/425495
Auto-Submit: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
diff --git a/internal/relui/buildrelease_test.go b/internal/relui/buildrelease_test.go
index 323b221..e096e58 100644
--- a/internal/relui/buildrelease_test.go
+++ b/internal/relui/buildrelease_test.go
@@ -19,7 +19,6 @@
"net/http/httptest"
"net/url"
"os"
- "os/exec"
"path/filepath"
"regexp"
"runtime"
@@ -32,12 +31,10 @@
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/go-github/github"
"github.com/google/uuid"
- "golang.org/x/build/buildlet"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal"
"golang.org/x/build/internal/gcsfs"
"golang.org/x/build/internal/task"
- "golang.org/x/build/internal/untar"
"golang.org/x/build/internal/workflow"
)
@@ -61,8 +58,9 @@
type releaseTestDeps struct {
ctx context.Context
- buildlets *fakeBuildlets
- gerrit *fakeGerrit
+ buildlets *task.FakeBuildlets
+ goRepo *task.FakeRepo
+ gerrit *reviewerCheckGerrit
versionTasks *task.VersionTasks
buildTasks *BuildReleaseTasks
milestoneTasks *task.MilestoneTasks
@@ -81,12 +79,7 @@
// Set up a server that will be used to serve inputs to the build.
bootstrapServer := httptest.NewServer(http.HandlerFunc(serveBootstrap))
t.Cleanup(bootstrapServer.Close)
- fakeBuildlets := &fakeBuildlets{
- t: t,
- dir: t.TempDir(),
- httpURL: bootstrapServer.URL,
- logs: map[string][]*[]string{},
- }
+ fakeBuildlets := task.NewFakeBuildlets(t, bootstrapServer.URL)
// Set up the fake signing process.
scratchDir := t.TempDir()
@@ -123,7 +116,13 @@
return nil
}
- gerrit := &fakeGerrit{createdTags: map[string]string{}}
+ goRepo := task.NewFakeRepo(t, "go")
+ base := goRepo.Commit(goFiles)
+ goRepo.Tag("go1.17", base)
+ dlRepo := task.NewFakeRepo(t, "dl")
+ fakeGerrit := task.NewFakeGerrit(t, goRepo, dlRepo)
+
+ gerrit := &reviewerCheckGerrit{FakeGerrit: fakeGerrit}
versionTasks := &task.VersionTasks{
Gerrit: gerrit,
GoProject: "go",
@@ -137,16 +136,14 @@
},
}
- snapshotServer := httptest.NewServer(http.HandlerFunc(serveSnapshot))
- t.Cleanup(snapshotServer.Close)
buildTasks := &BuildReleaseTasks{
GerritClient: gerrit,
GerritHTTPClient: http.DefaultClient,
- GerritURL: snapshotServer.URL,
+ GerritURL: fakeGerrit.GerritURL() + "/go",
GCSClient: nil,
ScratchURL: "file://" + filepath.ToSlash(scratchDir),
ServingURL: "file://" + filepath.ToSlash(servingDir),
- CreateBuildlet: fakeBuildlets.createBuildlet,
+ CreateBuildlet: fakeBuildlets.CreateBuildlet,
DownloadURL: dlServer.URL,
PublishFile: publishFile,
ApproveAction: func(ctx *workflow.TaskContext) error {
@@ -162,6 +159,7 @@
return &releaseTestDeps{
ctx: ctx,
buildlets: fakeBuildlets,
+ goRepo: goRepo,
gerrit: gerrit,
versionTasks: versionTasks,
buildTasks: buildTasks,
@@ -241,25 +239,30 @@
Kind: "installer",
}, "I'm a .pkg!\n")
- wantCLs := 2 // VERSION bump, DL
- if kind == task.KindBeta {
- wantCLs--
+ head, err := deps.gerrit.ReadBranchHead(deps.ctx, "dl", "master")
+ if err != nil {
+ t.Fatal(err)
}
- if deps.gerrit.changesCreated != wantCLs {
- t.Errorf("workflow sent %v changes to Gerrit, want %v", deps.gerrit.changesCreated, wantCLs)
+ content, err := deps.gerrit.ReadFile(deps.ctx, "dl", head, wantVersion+"/main.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(content), fmt.Sprintf("version.Run(%q)", wantVersion)) {
+ t.Errorf("unexpected dl content: %v", content)
}
- if len(deps.gerrit.createdTags) != 1 {
- t.Errorf("workflow created %v tags, want 1", deps.gerrit.createdTags)
+ tag, err := deps.gerrit.GetTag(deps.ctx, "go", wantVersion)
+ if err != nil {
+ t.Fatal(err)
}
- // TODO: consider logging this to golden files?
- for name, logs := range deps.buildlets.logs {
- t.Logf("%v buildlets:", name)
- for _, group := range logs {
- for _, line := range *group {
- t.Log(line)
- }
+ if kind != task.KindBeta {
+ version, err := deps.gerrit.ReadFile(deps.ctx, "go", tag.Revision, "VERSION")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(version) != wantVersion {
+ t.Errorf("VERSION file is %q, expected %q", version, wantVersion)
}
}
}
@@ -267,26 +270,20 @@
func testSecurity(t *testing.T, mergeFixes bool) {
deps := newReleaseTestDeps(t, "go1.18rc1")
- // Set up the fake merge process. Once we stop to ask for approval, switch
- // the public Gerrit server to serve the same content as the private server.
- approved := false
- privateServer := httptest.NewServer(http.HandlerFunc(serveSecureSnapshot))
- t.Cleanup(privateServer.Close)
- deps.buildTasks.PrivateGerritURL = privateServer.URL
-
- publicServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if approved && mergeFixes {
- serveSecureSnapshot(w, r)
- } else {
- serveSnapshot(w, r)
- }
- }))
- t.Cleanup(publicServer.Close)
- deps.buildTasks.GerritURL = publicServer.URL
+ // Set up the fake merge process. Once we stop to ask for approval, commit
+ // the fix to the public server.
+ privateRepo := task.NewFakeRepo(t, "go-private")
+ privateRepo.Commit(goFiles)
+ securityFix := map[string]string{"security.txt": "This file makes us secure"}
+ privateRef := privateRepo.Commit(securityFix)
+ privateGerrit := task.NewFakeGerrit(t, privateRepo)
+ deps.buildTasks.PrivateGerritURL = privateGerrit.GerritURL() + "/go-private"
defaultApprove := deps.buildTasks.ApproveAction
deps.buildTasks.ApproveAction = func(tc *workflow.TaskContext) error {
- approved = true
+ if mergeFixes {
+ deps.goRepo.Commit(securityFix)
+ }
return defaultApprove(tc)
}
@@ -297,7 +294,7 @@
w, err := workflow.Start(wd, map[string]interface{}{
"Targets to skip testing (or 'all') (optional)": []string{"js-wasm"},
- "Ref from the private repository to build from (optional)": "security-ref",
+ "Ref from the private repository to build from (optional)": privateRef,
})
if err != nil {
t.Fatal(err)
@@ -403,68 +400,21 @@
exit 0
`
+var goFiles = map[string]string{
+ "src/make.bash": makeScript,
+ "src/make.bat": makeScript,
+ "src/all.bash": allScript,
+ "src/all.bat": allScript,
+ "src/race.bash": allScript,
+ "src/race.bat": allScript,
+}
+
func serveBootstrap(w http.ResponseWriter, r *http.Request) {
- serveTarball("go-builder-data/go", map[string]string{
+ task.ServeTarball("go-builder-data/go", map[string]string{
"bin/go": "I'm a dummy bootstrap go command!",
}, w, r)
}
-func serveSnapshot(w http.ResponseWriter, r *http.Request) {
- serveTarball("+archive", map[string]string{
- "src/make.bash": makeScript,
- "src/make.bat": makeScript,
- "src/all.bash": allScript,
- "src/all.bat": allScript,
- "src/race.bash": allScript,
- "src/race.bat": allScript,
- }, w, r)
-}
-
-func serveSecureSnapshot(w http.ResponseWriter, r *http.Request) {
- serveTarball("+archive", map[string]string{
- "src/make.bash": makeScript,
- "src/make.bat": makeScript,
- "src/all.bash": allScript,
- "src/all.bat": allScript,
- "src/race.bash": allScript,
- "src/race.bat": allScript,
- "security.txt": "This file makes us secure",
- }, w, r)
-}
-
-// serveTarballs serves the files the release process relies on.
-// PutTarFromURL is hardcoded to read from this server.
-func serveTarball(pathMatch string, files map[string]string, w http.ResponseWriter, r *http.Request) {
- if !strings.Contains(r.URL.Path, pathMatch) {
- w.WriteHeader(http.StatusNotFound)
- return
- }
-
- gzw := gzip.NewWriter(w)
- tw := tar.NewWriter(gzw)
-
- for name, contents := range files {
- if err := tw.WriteHeader(&tar.Header{
- Typeflag: tar.TypeReg,
- Name: name,
- Size: int64(len(contents)),
- Mode: 0777,
- }); err != nil {
- panic(err)
- }
- if _, err := tw.Write([]byte(contents)); err != nil {
- panic(err)
- }
- }
-
- if err := tw.Close(); err != nil {
- panic(err)
- }
- if err := gzw.Close(); err != nil {
- panic(err)
- }
-}
-
func checkFile(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, check func(*testing.T, []byte)) {
t.Run(filename, func(t *testing.T) {
f, ok := files[filename]
@@ -558,36 +508,16 @@
})
}
-type fakeGerrit struct {
- changesCreated int
- createdTags map[string]string
- wantReviewers []string
- task.GerritClient
+type reviewerCheckGerrit struct {
+ wantReviewers []string
+ *task.FakeGerrit
}
-func (g *fakeGerrit) CreateAutoSubmitChange(ctx context.Context, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {
- g.changesCreated++
+func (g *reviewerCheckGerrit) CreateAutoSubmitChange(ctx context.Context, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {
if diff := cmp.Diff(g.wantReviewers, reviewers, cmpopts.EquateEmpty()); diff != "" {
return "", fmt.Errorf("unexpected reviewers for CL: %v", diff)
}
- return "fake~12345", nil
-}
-
-func (g *fakeGerrit) Submitted(ctx context.Context, changeID, baseCommit string) (string, bool, error) {
- return "fakehash", true, nil
-}
-
-func (g *fakeGerrit) ListTags(ctx context.Context, project string) ([]string, error) {
- return []string{"go1.17"}, nil
-}
-
-func (g *fakeGerrit) Tag(ctx context.Context, project, tag, commit string) error {
- g.createdTags[tag] = commit
- return nil
-}
-
-func (g *fakeGerrit) ReadBranchHead(ctx context.Context, project, branch string) (string, error) {
- return fmt.Sprintf("fake HEAD commit for %v/%v", project, branch), nil
+ return g.FakeGerrit.CreateAutoSubmitChange(ctx, input, reviewers, contents)
}
type fakeGitHub struct {
@@ -609,186 +539,6 @@
return nil, nil, nil
}
-type fakeBuildlets struct {
- t *testing.T
- dir string
- httpURL string
-
- mu sync.Mutex
- nextID int
- logs map[string][]*[]string
-}
-
-func (b *fakeBuildlets) createBuildlet(_ context.Context, kind string) (buildlet.RemoteClient, error) {
- b.mu.Lock()
- buildletDir := filepath.Join(b.dir, kind, fmt.Sprint(b.nextID))
- logs := &[]string{}
- b.nextID++
- b.logs[kind] = append(b.logs[kind], logs)
- b.mu.Unlock()
- logf := func(format string, args ...interface{}) {
- line := fmt.Sprintf(format, args...)
- line = strings.ReplaceAll(line, buildletDir, "$WORK")
- *logs = append(*logs, line)
- }
- logf("--- create buildlet ---")
-
- return &fakeBuildlet{
- t: b.t,
- kind: kind,
- dir: buildletDir,
- httpURL: b.httpURL,
- logf: logf,
- }, nil
-}
-
-type fakeBuildlet struct {
- buildlet.Client
- t *testing.T
- kind string
- dir string
- httpURL string
- logf func(string, ...interface{})
- closed bool
-}
-
-func (b *fakeBuildlet) Close() error {
- if !b.closed {
- b.logf("--- destroy buildlet ---")
- b.closed = true
- }
- return nil
-}
-
-func (b *fakeBuildlet) Exec(ctx context.Context, cmd string, opts buildlet.ExecOpts) (remoteErr error, execErr error) {
- b.logf("exec %v %v\n\twd %q env %v", cmd, opts.Args, opts.Dir, opts.ExtraEnv)
- absCmd := filepath.Join(b.dir, cmd)
-retry:
- c := exec.CommandContext(ctx, absCmd, opts.Args...)
- c.Env = append(os.Environ(), opts.ExtraEnv...)
- buf := &bytes.Buffer{}
- var w io.Writer = buf
- if opts.Output != nil {
- w = io.MultiWriter(w, opts.Output)
- }
- c.Stdout = w
- c.Stderr = w
- if opts.Dir == "" {
- c.Dir = filepath.Dir(absCmd)
- } else {
- c.Dir = filepath.Join(b.dir, opts.Dir)
- }
- err := c.Run()
- // Work around Unix foolishness. See go.dev/issue/22315.
- if err != nil && strings.Contains(err.Error(), "text file busy") {
- time.Sleep(100 * time.Millisecond)
- goto retry
- }
- if err != nil {
- return nil, fmt.Errorf("command %v %v failed: %v output: %q", cmd, opts.Args, err, buf.String())
- }
- return nil, nil
-}
-
-func (b *fakeBuildlet) GetTar(ctx context.Context, dir string) (io.ReadCloser, error) {
- b.logf("get tar of %q", dir)
- buf := &bytes.Buffer{}
- zw := gzip.NewWriter(buf)
- tw := tar.NewWriter(zw)
- base := filepath.Join(b.dir, filepath.FromSlash(dir))
- // Copied pretty much wholesale from buildlet.go.
- err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- rel := strings.TrimPrefix(filepath.ToSlash(strings.TrimPrefix(path, base)), "/")
- th, err := tar.FileInfoHeader(fi, path)
- if err != nil {
- return err
- }
- th.Name = rel
- if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
- th.Name += "/"
- }
- if th.Name == "/" {
- return nil
- }
- if err := tw.WriteHeader(th); err != nil {
- return err
- }
- if fi.Mode().IsRegular() {
- f, err := os.Open(path)
- if err != nil {
- return err
- }
- defer f.Close()
- if _, err := io.Copy(tw, f); err != nil {
- return err
- }
- }
- return nil
- })
- if err != nil {
- return nil, err
- }
- if err := tw.Close(); err != nil {
- return nil, err
- }
- if err := zw.Close(); err != nil {
- return nil, err
- }
- return ioutil.NopCloser(buf), nil
-}
-
-func (b *fakeBuildlet) ListDir(ctx context.Context, dir string, opts buildlet.ListDirOpts, fn func(buildlet.DirEntry)) error {
- // We call this when something goes wrong, so we need it to "succeed".
- // It's not worth implementing; return some nonsense.
- fn(buildlet.DirEntry{
- Line: "ListDir is silently unimplemented, sorry",
- })
- return nil
-}
-
-func (b *fakeBuildlet) Put(ctx context.Context, r io.Reader, path string, mode os.FileMode) error {
- b.logf("write file %q with mode %0o", path, mode)
- f, err := os.OpenFile(filepath.Join(b.dir, path), os.O_CREATE|os.O_RDWR, mode)
- if err != nil {
- return err
- }
- defer f.Close()
-
- if _, err := io.Copy(f, r); err != nil {
- return err
- }
- return f.Close()
-}
-
-func (b *fakeBuildlet) PutTar(ctx context.Context, r io.Reader, dir string) error {
- b.logf("put tar to %q", dir)
- return untar.Untar(r, filepath.Join(b.dir, dir))
-}
-
-func (b *fakeBuildlet) PutTarFromURL(ctx context.Context, tarURL string, dir string) error {
- b.logf("put tar from %v to %q", tarURL, dir)
- u, err := url.Parse(tarURL)
- if err != nil {
- return err
- }
- resp, err := http.Get(b.httpURL + u.Path)
- if err != nil {
- return err
- }
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("unexpected status for %q: %v", tarURL, resp.Status)
- }
- defer resp.Body.Close()
- return untar.Untar(resp.Body, filepath.Join(b.dir, dir))
-}
-
-func (b *fakeBuildlet) WorkDir(ctx context.Context) (string, error) {
- return b.dir, nil
-}
-
type verboseListener struct {
t *testing.T
outputListener func(string, interface{})
diff --git a/internal/task/fakes.go b/internal/task/fakes.go
new file mode 100644
index 0000000..0115090
--- /dev/null
+++ b/internal/task/fakes.go
@@ -0,0 +1,432 @@
+package task
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "golang.org/x/build/buildlet"
+ "golang.org/x/build/gerrit"
+ "golang.org/x/build/internal/untar"
+)
+
+// ServeTarball serves files as a .tar.gz to w, only if path contains pathMatch.
+func ServeTarball(pathMatch string, files map[string]string, w http.ResponseWriter, r *http.Request) {
+ if !strings.Contains(r.URL.Path, pathMatch) {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+
+ gzw := gzip.NewWriter(w)
+ tw := tar.NewWriter(gzw)
+
+ for name, contents := range files {
+ if err := tw.WriteHeader(&tar.Header{
+ Typeflag: tar.TypeReg,
+ Name: name,
+ Size: int64(len(contents)),
+ Mode: 0777,
+ }); err != nil {
+ panic(err)
+ }
+ if _, err := tw.Write([]byte(contents)); err != nil {
+ panic(err)
+ }
+ }
+
+ if err := tw.Close(); err != nil {
+ panic(err)
+ }
+ if err := gzw.Close(); err != nil {
+ panic(err)
+ }
+}
+
+func NewFakeBuildlets(t *testing.T, httpServer string) *FakeBuildlets {
+ return &FakeBuildlets{
+ t: t,
+ dir: t.TempDir(),
+ httpURL: httpServer,
+ logs: map[string][]*[]string{},
+ }
+}
+
+type FakeBuildlets struct {
+ t *testing.T
+ dir string
+ httpURL string
+
+ mu sync.Mutex
+ nextID int
+ logs map[string][]*[]string
+}
+
+func (b *FakeBuildlets) CreateBuildlet(_ context.Context, kind string) (buildlet.RemoteClient, error) {
+ b.mu.Lock()
+ buildletDir := filepath.Join(b.dir, kind, fmt.Sprint(b.nextID))
+ logs := &[]string{}
+ b.nextID++
+ b.logs[kind] = append(b.logs[kind], logs)
+ b.mu.Unlock()
+ logf := func(format string, args ...interface{}) {
+ line := fmt.Sprintf(format, args...)
+ line = strings.ReplaceAll(line, buildletDir, "$WORK")
+ *logs = append(*logs, line)
+ }
+ logf("--- create buildlet ---")
+
+ return &fakeBuildlet{
+ t: b.t,
+ kind: kind,
+ dir: buildletDir,
+ httpURL: b.httpURL,
+ logf: logf,
+ }, nil
+}
+
+func (b *FakeBuildlets) DumpLogs() {
+ for name, logs := range b.logs {
+ b.t.Logf("%v buildlets:", name)
+ for _, group := range logs {
+ for _, line := range *group {
+ b.t.Log(line)
+ }
+ }
+ }
+}
+
+type fakeBuildlet struct {
+ buildlet.Client
+ t *testing.T
+ kind string
+ dir string
+ httpURL string
+ logf func(string, ...interface{})
+ closed bool
+}
+
+func (b *fakeBuildlet) Close() error {
+ if !b.closed {
+ b.logf("--- destroy buildlet ---")
+ b.closed = true
+ }
+ return nil
+}
+
+func (b *fakeBuildlet) Exec(ctx context.Context, cmd string, opts buildlet.ExecOpts) (remoteErr error, execErr error) {
+ b.logf("exec %v %v\n\twd %q env %v", cmd, opts.Args, opts.Dir, opts.ExtraEnv)
+ if !strings.HasPrefix(cmd, "/") && !opts.SystemLevel {
+ cmd = filepath.Join(b.dir, cmd)
+ }
+retry:
+ c := exec.CommandContext(ctx, cmd, opts.Args...)
+ c.Env = append(os.Environ(), opts.ExtraEnv...)
+ buf := &bytes.Buffer{}
+ var w io.Writer = buf
+ if opts.Output != nil {
+ w = io.MultiWriter(w, opts.Output)
+ }
+ c.Stdout = w
+ c.Stderr = w
+ if opts.Dir == "" {
+ c.Dir = filepath.Dir(cmd)
+ } else {
+ c.Dir = filepath.Join(b.dir, opts.Dir)
+ }
+ err := c.Run()
+ // Work around Unix foolishness. See go.dev/issue/22315.
+ if err != nil && strings.Contains(err.Error(), "text file busy") {
+ time.Sleep(100 * time.Millisecond)
+ goto retry
+ }
+ if err != nil {
+ return nil, fmt.Errorf("command %v %v failed: %v output: %q", cmd, opts.Args, err, buf.String())
+ }
+ return nil, nil
+}
+
+func (b *fakeBuildlet) GetTar(ctx context.Context, dir string) (io.ReadCloser, error) {
+ b.logf("get tar of %q", dir)
+ buf := &bytes.Buffer{}
+ zw := gzip.NewWriter(buf)
+ tw := tar.NewWriter(zw)
+ base := filepath.Join(b.dir, filepath.FromSlash(dir))
+ // Copied pretty much wholesale from buildlet.go.
+ err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ rel := strings.TrimPrefix(filepath.ToSlash(strings.TrimPrefix(path, base)), "/")
+ th, err := tar.FileInfoHeader(fi, path)
+ if err != nil {
+ return err
+ }
+ th.Name = rel
+ if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
+ th.Name += "/"
+ }
+ if th.Name == "/" {
+ return nil
+ }
+ if err := tw.WriteHeader(th); err != nil {
+ return err
+ }
+ if fi.Mode().IsRegular() {
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if _, err := io.Copy(tw, f); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ if err := tw.Close(); err != nil {
+ return nil, err
+ }
+ if err := zw.Close(); err != nil {
+ return nil, err
+ }
+ return ioutil.NopCloser(buf), nil
+}
+
+func (b *fakeBuildlet) ListDir(ctx context.Context, dir string, opts buildlet.ListDirOpts, fn func(buildlet.DirEntry)) error {
+ // We call this when something goes wrong, so we need it to "succeed".
+ // It's not worth implementing; return some nonsense.
+ fn(buildlet.DirEntry{
+ Line: "ListDir is silently unimplemented, sorry",
+ })
+ return nil
+}
+
+func (b *fakeBuildlet) Put(ctx context.Context, r io.Reader, path string, mode os.FileMode) error {
+ b.logf("write file %q with mode %0o", path, mode)
+ f, err := os.OpenFile(filepath.Join(b.dir, path), os.O_CREATE|os.O_RDWR, mode)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ if _, err := io.Copy(f, r); err != nil {
+ return err
+ }
+ return f.Close()
+}
+
+func (b *fakeBuildlet) PutTar(ctx context.Context, r io.Reader, dir string) error {
+ b.logf("put tar to %q", dir)
+ return untar.Untar(r, filepath.Join(b.dir, dir))
+}
+
+func (b *fakeBuildlet) PutTarFromURL(ctx context.Context, tarURL string, dir string) error {
+ url, err := url.Parse(tarURL)
+ if err != nil {
+ return err
+ }
+ rewritten := url.String()
+ if !strings.Contains(url.Host, "localhost") && !strings.Contains(url.Host, "127.0.0.1") {
+ rewritten = b.httpURL + url.Path
+ }
+ b.logf("put tar from %v (rewritten to %v) to %q", tarURL, rewritten, dir)
+
+ resp, err := http.Get(rewritten)
+ if err != nil {
+ return err
+ }
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("unexpected status for %q: %v", tarURL, resp.Status)
+ }
+ defer resp.Body.Close()
+ return untar.Untar(resp.Body, filepath.Join(b.dir, dir))
+}
+
+func (b *fakeBuildlet) WorkDir(ctx context.Context) (string, error) {
+ return b.dir, nil
+}
+
+func NewFakeGerrit(t *testing.T, repos ...*FakeRepo) *FakeGerrit {
+ result := &FakeGerrit{
+ repos: map[string]*FakeRepo{},
+ }
+ server := httptest.NewServer(http.HandlerFunc(result.serveHTTP))
+ result.serverURL = server.URL
+ t.Cleanup(server.Close)
+
+ for _, r := range repos {
+ result.repos[r.name] = r
+ }
+ return result
+}
+
+type FakeGerrit struct {
+ repos map[string]*FakeRepo
+ serverURL string
+}
+
+type FakeRepo struct {
+ t *testing.T
+ name string
+ seq int
+ history []string // oldest to newest
+ content map[string]map[string]string
+ tags map[string]string
+}
+
+func NewFakeRepo(t *testing.T, name string) *FakeRepo {
+ return &FakeRepo{
+ t: t,
+ name: name,
+ content: map[string]map[string]string{},
+ tags: map[string]string{},
+ }
+}
+
+func (repo *FakeRepo) Commit(contents map[string]string) string {
+ rev := fmt.Sprintf("%v~%v", repo.name, repo.seq)
+ repo.seq++
+
+ newContent := map[string]string{}
+ if len(repo.history) != 0 {
+ for k, v := range repo.content[repo.history[len(repo.history)-1]] {
+ newContent[k] = v
+ }
+ }
+ for k, v := range contents {
+ newContent[k] = v
+ }
+ repo.content[rev] = newContent
+ repo.history = append(repo.history, rev)
+ return rev
+}
+
+func (repo *FakeRepo) Tag(tag, commit string) {
+ if _, ok := repo.content[commit]; !ok {
+ repo.t.Fatalf("commit %q does not exist on repo %q", commit, repo.name)
+ }
+ if _, ok := repo.tags[tag]; ok {
+ repo.t.Fatalf("tag %q already exists on repo %q", commit, repo.name)
+ }
+ repo.tags[tag] = commit
+}
+
+var _ GerritClient = (*FakeGerrit)(nil)
+
+func (g *FakeGerrit) ListProjects(ctx context.Context) ([]string, error) {
+ var names []string
+ for k := range g.repos {
+ names = append(names, k)
+ }
+ return names, nil
+}
+
+func (g *FakeGerrit) repo(name string) (*FakeRepo, error) {
+ if r, ok := g.repos[name]; ok {
+ return r, nil
+ } else {
+ return nil, fmt.Errorf("no such repo %v: %w", name, gerrit.ErrResourceNotExist)
+ }
+}
+
+func (g *FakeGerrit) ReadBranchHead(ctx context.Context, project, branch string) (string, error) {
+ repo, err := g.repo(project)
+ if err != nil {
+ return "", err
+ }
+ return repo.history[len(repo.history)-1], nil
+}
+
+func (g *FakeGerrit) ReadFile(ctx context.Context, project string, commit string, file string) ([]byte, error) {
+ repo, err := g.repo(project)
+ if err != nil {
+ return nil, err
+ }
+ content := repo.content[commit][file]
+ if content == "" {
+ return nil, fmt.Errorf("commit/file not found %v at %v: %w", file, commit, gerrit.ErrResourceNotExist)
+ }
+ return []byte(content), nil
+}
+
+func (g *FakeGerrit) ListTags(ctx context.Context, project string) ([]string, error) {
+ repo, err := g.repo(project)
+ if err != nil {
+ return nil, err
+ }
+ var tags []string
+ for k := range repo.tags {
+ tags = append(tags, k)
+ }
+ return tags, nil
+}
+
+func (g *FakeGerrit) GetTag(ctx context.Context, project string, tag string) (gerrit.TagInfo, error) {
+ repo, err := g.repo(project)
+ if err != nil {
+ return gerrit.TagInfo{}, err
+ }
+ if commit, ok := repo.tags[tag]; ok {
+ return gerrit.TagInfo{Revision: commit}, nil
+ } else {
+ return gerrit.TagInfo{}, fmt.Errorf("tag not found: %w", gerrit.ErrResourceNotExist)
+ }
+}
+
+func (g *FakeGerrit) CreateAutoSubmitChange(ctx context.Context, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {
+ repo, err := g.repo(input.Project)
+ if err != nil {
+ return "", err
+ }
+ commit := repo.Commit(contents)
+ return "cl_" + commit, nil
+}
+
+func (g *FakeGerrit) Submitted(ctx context.Context, changeID, baseCommit string) (string, bool, error) {
+ return strings.TrimPrefix(changeID, "cl_"), true, nil
+}
+
+func (g *FakeGerrit) Tag(ctx context.Context, project, tag, commit string) error {
+ repo, err := g.repo(project)
+ if err != nil {
+ return err
+ }
+ repo.Tag(tag, commit)
+ return nil
+}
+
+func (g *FakeGerrit) GerritURL() string {
+ return g.serverURL
+}
+
+func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, r *http.Request) {
+ parts := strings.Split(r.URL.Path, "/")
+ if len(parts) != 4 {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ repo, err := g.repo(parts[1])
+ if err != nil {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ rev := strings.TrimSuffix(parts[3], ".tar.gz")
+ ServeTarball("", repo.content[rev], w, r)
+}
diff --git a/internal/task/milestones_test.go b/internal/task/milestones_test.go
index 68b30aa..5952d3b 100644
--- a/internal/task/milestones_test.go
+++ b/internal/task/milestones_test.go
@@ -23,7 +23,7 @@
func TestMilestones(t *testing.T) {
ctx := &workflow.TaskContext{
Context: context.Background(),
- Logger: &testLogger{t},
+ Logger: &testLogger{t, ""},
}
if !*flagRun {
@@ -142,11 +142,3 @@
})
return normal, blocker, err
}
-
-type testLogger struct {
- t *testing.T
-}
-
-func (l *testLogger) Printf(format string, v ...interface{}) {
- l.t.Logf("LOG: %s", fmt.Sprintf(format, v...))
-}
diff --git a/internal/task/tagx.go b/internal/task/tagx.go
index 59f745f..ea74859 100644
--- a/internal/task/tagx.go
+++ b/internal/task/tagx.go
@@ -21,8 +21,10 @@
)
type TagXReposTasks struct {
- Gerrit GerritClient
- CreateBuildlet func(context.Context, string) (buildlet.RemoteClient, error)
+ Gerrit GerritClient
+ GerritURL string
+ CreateBuildlet func(context.Context, string) (buildlet.RemoteClient, error)
+ LatestGoBinaries func(context.Context) (string, error)
}
func (x *TagXReposTasks) NewDefinition() *wf.Definition {
@@ -248,7 +250,7 @@
}
func (x *TagXReposTasks) UpdateGoMod(ctx *wf.TaskContext, repo TagRepo, deps []TagRepo, commit string) (UpdatedModSum, error) {
- binaries, err := latestGoBinaries(ctx)
+ binaries, err := x.LatestGoBinaries(ctx)
if err != nil {
return UpdatedModSum{}, err
}
@@ -261,8 +263,8 @@
if err := bc.PutTarFromURL(ctx, binaries, ""); err != nil {
return UpdatedModSum{}, err
}
- archive := fmt.Sprintf("https://go.googlesource.com/%v/+archive/%v.tar.gz", repo.Name, commit)
- if err := bc.PutTarFromURL(ctx, archive, "repo"); err != nil {
+ tarURL := fmt.Sprintf("%s/%s/+archive/%s.tar.gz", x.GerritURL, repo.Name, commit)
+ if err := bc.PutTarFromURL(ctx, tarURL, "repo"); err != nil {
return UpdatedModSum{}, err
}
@@ -322,7 +324,7 @@
return UpdatedModSum{mod.String(), sum.String()}, nil
}
-func latestGoBinaries(ctx *wf.TaskContext) (string, error) {
+func LatestGoBinaries(ctx context.Context) (string, error) {
resp, err := ctxhttp.Get(ctx, http.DefaultClient, "https://go.dev/dl/?mode=json")
if err != nil {
return "", err
@@ -347,9 +349,6 @@
}
func (x *TagXReposTasks) MailGoMod(ctx *wf.TaskContext, repo string, gomod UpdatedModSum) (string, error) {
- if dryRun {
- return "", fmt.Errorf("nope")
- }
return x.Gerrit.CreateAutoSubmitChange(ctx, gerrit.ChangeInput{
Project: repo,
Branch: "master",
@@ -364,8 +363,6 @@
return nil
}
-const dryRun = true
-
// MaybeTag tags repo at commit with the next version, unless commit is already
// the latest tagged version. repo is returned with Version populated.
func (x *TagXReposTasks) MaybeTag(ctx *wf.TaskContext, repo TagRepo, commit string) (TagRepo, error) {
@@ -380,9 +377,8 @@
highestRelease = tag
}
}
- if highestRelease == "" || dryRun {
- repo.Version = "latest"
- return repo, nil
+ if highestRelease == "" {
+ return TagRepo{}, fmt.Errorf("no semver tags found in %v", tags)
}
tagInfo, err := x.Gerrit.GetTag(ctx, repo.Name, highestRelease)
diff --git a/internal/task/tagx_test.go b/internal/task/tagx_test.go
index 4a3e5aa..e19083b 100644
--- a/internal/task/tagx_test.go
+++ b/internal/task/tagx_test.go
@@ -3,10 +3,17 @@
import (
"context"
"flag"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "runtime"
"strings"
"testing"
+ "time"
"github.com/google/go-cmp/cmp"
+ "github.com/google/uuid"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/workflow"
)
@@ -25,7 +32,7 @@
}
ctx := &workflow.TaskContext{
Context: context.Background(),
- Logger: &testLogger{t},
+ Logger: &testLogger{t, ""},
}
repos, err := tasks.SelectRepos(ctx)
if err != nil {
@@ -116,3 +123,145 @@
}
}
}
+
+const fakeGo = `#!/bin/bash -eu
+
+case "$1" in
+"get")
+ ls go.mod go.sum >/dev/null
+ for i in "${@:2}"; do
+ echo -e "// pretend we've upgraded to $i" >> go.mod
+ echo "$i h1:asdasd" | tr '@' ' ' >> go.sum
+ done
+ ;;
+"mod")
+ ls go.mod go.sum >/dev/null
+ echo "tidied!" >> go.mod
+ ;;
+*)
+ echo unexpected command $@
+ exit 1
+ ;;
+esac
+`
+
+func TestTagXRepos(t *testing.T) {
+ if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+ t.Skip("Requires bash shell scripting support.")
+ }
+
+ goServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ ServeTarball("dl/go1.19.linux-amd64.tar.gz", map[string]string{
+ "go/bin/go": fakeGo,
+ }, w, r)
+ }))
+ t.Cleanup(goServer.Close)
+
+ fakeBuildlets := NewFakeBuildlets(t, "")
+
+ sys := NewFakeRepo(t, "sys")
+ sys1 := sys.Commit(map[string]string{
+ "go.mod": "module golang.org/x/sys\n",
+ "go.sum": "\n",
+ })
+ sys.Tag("v0.1.0", sys1)
+ sys2 := sys.Commit(map[string]string{
+ "main.go": "package main",
+ })
+ mod := NewFakeRepo(t, "mod")
+ mod1 := mod.Commit(map[string]string{
+ "go.mod": "module golang.org/x/mod\n",
+ "go.sum": "\n",
+ })
+ mod.Tag("v1.0.0", mod1)
+ tools := NewFakeRepo(t, "tools")
+ tools1 := tools.Commit(map[string]string{
+ "go.mod": "module golang.org/x/tools\nrequire golang.org/x/mod v1.0.0\nrequire golang.org/x/sys v0.1.0\n",
+ "go.sum": "\n",
+ })
+ tools.Tag("v1.1.5", tools1)
+ fakeGerrit := NewFakeGerrit(t, sys, mod, tools)
+ tasks := &TagXReposTasks{
+ Gerrit: fakeGerrit,
+ GerritURL: fakeGerrit.GerritURL(),
+ CreateBuildlet: fakeBuildlets.CreateBuildlet,
+ LatestGoBinaries: func(context.Context) (string, error) {
+ return goServer.URL + "/dl/go1.19.linux-amd64.tar.gz", nil
+ },
+ }
+ wd := tasks.NewDefinition()
+ w, err := workflow.Start(wd, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _, err = w.Run(ctx, &verboseListener{t: t})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tag, err := fakeGerrit.GetTag(ctx, "sys", "v0.2.0")
+ if err != nil {
+ t.Fatalf("sys should have been tagged with v0.2.0: %v", err)
+ }
+ if tag.Revision != sys2 {
+ t.Errorf("sys v0.2.0 = %v, want %v", tag.Revision, sys2)
+ }
+
+ tags, err := fakeGerrit.ListTags(ctx, "mod")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(tags, []string{"v1.0.0"}) {
+ t.Errorf("mod has tags %v, wanted only v1.0.0", tags)
+ }
+
+ tag, err = fakeGerrit.GetTag(ctx, "tools", "v1.2.0")
+ if err != nil {
+ t.Fatalf("tools should have been tagged with v1.2.0: %v", err)
+ }
+ goMod, err := fakeGerrit.ReadFile(ctx, "tools", tag.Revision, "go.mod")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(goMod), "sys@v0.2.0") || !strings.Contains(string(goMod), "mod@v1.0.0") {
+ t.Errorf("tools should use sys v0.2.0 and mod v1.0.0. go.mod: %v", string(goMod))
+ }
+ if !strings.Contains(string(goMod), "tidied!") {
+ t.Error("tools go.mod should be tidied")
+ }
+}
+
+type verboseListener struct {
+ t *testing.T
+ outputListener func(string, interface{})
+}
+
+func (l *verboseListener) TaskStateChanged(_ uuid.UUID, _ string, st *workflow.TaskState) error {
+ switch {
+ case !st.Finished:
+ l.t.Logf("task %-10v: started", st.Name)
+ case st.Error != "":
+ l.t.Logf("task %-10v: error: %v", st.Name, st.Error)
+ default:
+ l.t.Logf("task %-10v: done: %v", st.Name, st.Result)
+ if l.outputListener != nil {
+ l.outputListener(st.Name, st.Result)
+ }
+ }
+ return nil
+}
+
+func (l *verboseListener) Logger(_ uuid.UUID, task string) workflow.Logger {
+ return &testLogger{t: l.t, task: task}
+}
+
+type testLogger struct {
+ t *testing.T
+ task string // Optional.
+}
+
+func (l *testLogger) Printf(format string, v ...interface{}) {
+ l.t.Logf("task %-10v: LOG: %s", l.task, fmt.Sprintf(format, v...))
+}
diff --git a/internal/task/version_test.go b/internal/task/version_test.go
index 512973e..6d035cb 100644
--- a/internal/task/version_test.go
+++ b/internal/task/version_test.go
@@ -25,7 +25,7 @@
}
ctx := &workflow.TaskContext{
Context: context.Background(),
- Logger: &testLogger{t},
+ Logger: &testLogger{t, ""},
}
versions := map[ReleaseKind]string{}
@@ -53,7 +53,7 @@
}
ctx := &workflow.TaskContext{
Context: context.Background(),
- Logger: &testLogger{t},
+ Logger: &testLogger{t, ""},
}
versions := map[ReleaseKind]string{}
for kind := ReleaseKind(1); kind <= KindPrevMinor; kind++ {
@@ -95,7 +95,7 @@
}
ctx := &workflow.TaskContext{
Context: context.Background(),
- Logger: &testLogger{t},
+ Logger: &testLogger{t, ""},
}
changeID, err := tasks.CreateAutoSubmitVersionCL(ctx, "master", nil, "version string")
