cmd/gitmirror: add tests
Add integration tests. They're on the slow side, as much as a second per
test, but the coverage is pretty solid and I hope they'll be easy to
read and maintain.
The tests run a real mirror, but without the normal driver loop and with
the repositories redirected to point to the local filesystem. That costs
us coverage of things like the gerrit polling, but removes the timing
sensitivity. I think that's a reasonable tradeoff; we'd just be mocking
maintner and gerrit anyway.
Also fix a bug that the tests surfaced: we can't run clone in the
directory we want it to create.
Change-Id: I8fa319b9bd3de194e75a2fdc2ee5a5cef71ec1aa
Reviewed-on: https://go-review.googlesource.com/c/build/+/324762
Trust: Heschi Kreinick <heschi@google.com>
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/cmd/gitmirror/gitmirror.go b/cmd/gitmirror/gitmirror.go
index 38f3b8d..9ba6984 100644
--- a/cmd/gitmirror/gitmirror.go
+++ b/cmd/gitmirror/gitmirror.go
@@ -101,7 +101,7 @@
}
for _, repo := range m.repos {
- go repo.Loop()
+ go repo.loop()
}
go m.pollGerritAndTickle()
go m.subscribeToMaintnerAndTickleLoop()
@@ -218,7 +218,7 @@
func (m *mirror) addMirrors() error {
for _, repo := range m.repos {
if repo.meta.MirrorToGitHub {
- if err := repo.addRemote("github", "git@github.com:"+repo.meta.GitHubRepo()+".git"); err != nil {
+ if err := repo.addRemote("github", "git@github.com:"+repo.meta.GitHubRepo+".git"); err != nil {
return fmt.Errorf("adding GitHub remote: %v", err)
}
}
@@ -346,7 +346,7 @@
if err == nil {
r.logf("ran git %s in %v", args, time.Since(start))
} else {
- r.logf("git %s failed after %v: %v\n%v", args, time.Since(start), err, string(stderr))
+ r.logf("git %s failed after %v: %v\nstdout: %v\nstderr: %v\n", args, time.Since(start), err, string(stdout), string(stderr))
}
return stdout, stderr, err
}
@@ -357,7 +357,12 @@
stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
cmd := exec.CommandContext(ctx, "git", args...)
- cmd.Dir = r.root
+ if args[0] == "clone" {
+ // Small hack: if we're cloning, the root doesn't exist yet.
+ cmd.Dir = "/"
+ } else {
+ cmd.Dir = r.root
+ }
cmd.Env = append(os.Environ(), "HOME="+r.mirror.homeDir)
cmd.Stdout, cmd.Stderr = stdout, stderr
err := cmd.Run()
@@ -429,28 +434,15 @@
return ioutil.WriteFile(filepath.Join(r.root, "remotes", name), []byte(remote), 0777)
}
-// Loop continuously runs "git fetch" in the repo, checks for new
+// loop continuously runs "git fetch" in the repo, checks for new
// commits and mirrors commits to a destination repo (if enabled).
-func (r *repo) Loop() {
-outer:
+func (r *repo) loop() {
for {
- if err := r.fetch(); err != nil {
- r.logf("fetch failed in repo loop: %v", err)
- r.setErr(err)
+ if err := r.loopOnce(); err != nil {
time.Sleep(10 * time.Second)
continue
}
- for _, dest := range r.dests {
- if err := r.push(dest); err != nil {
- r.logf("push failed in repo loop: %v", err)
- r.setErr(err)
- time.Sleep(10 * time.Second)
- continue outer
- }
- }
- r.setErr(nil)
- r.setStatus("waiting")
// We still run a timer but a very slow one, just
// in case the mechanism updating the repo tickler
// breaks for some reason.
@@ -465,6 +457,24 @@
}
}
+func (r *repo) loopOnce() error {
+ if err := r.fetch(); err != nil {
+ r.logf("fetch failed: %v", err)
+ r.setErr(err)
+ return err
+ }
+ for _, dest := range r.dests {
+ if err := r.push(dest); err != nil {
+ r.logf("push failed: %v", err)
+ r.setErr(err)
+ return err
+ }
+ }
+ r.setErr(nil)
+ r.setStatus("waiting")
+ return nil
+}
+
func (r *repo) logf(format string, args ...interface{}) {
log.Printf(r.name+": "+format, args...)
}
diff --git a/cmd/gitmirror/gitmirror_test.go b/cmd/gitmirror/gitmirror_test.go
index 4548ba5..f8104eb 100644
--- a/cmd/gitmirror/gitmirror_test.go
+++ b/cmd/gitmirror/gitmirror_test.go
@@ -5,65 +5,33 @@
package main
import (
+ "bytes"
+ "fmt"
"io/ioutil"
- "log"
+ "net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
+
+ repospkg "golang.org/x/build/repos"
)
-func TestMain(m *testing.M) {
- // The tests need a dummy directory that exists with a
- // basename of "build", but the tests never write to it. So we
- // can create one and share it for all tests.
- tempDir, err := ioutil.TempDir("", "")
- if err != nil {
- log.Fatal(err)
- }
- tempRepoRoot = filepath.Join(tempDir, "build")
- if err := os.Mkdir(tempRepoRoot, 0700); err != nil {
- log.Fatal(err)
- }
-
- e := m.Run()
- os.RemoveAll(tempDir)
- os.Exit(e)
-}
-
-var tempRepoRoot string
-
-func newTestRepo() *repo {
- return &repo{
- name: "build",
- root: tempRepoRoot,
- }
-}
-
func TestHomepage(t *testing.T) {
- req := httptest.NewRequest("GET", "/", nil)
- w := httptest.NewRecorder()
- (&mirror{}).handleRoot(w, req)
- if w.Code != 200 {
- t.Fatalf("GET /: want code 200, got %d", w.Code)
- }
- if hdr := w.Header().Get("Content-Type"); !strings.Contains(hdr, "text/html") {
- t.Fatalf("GET /: want html content-type, got %s", hdr)
+ tm := newTestMirror(t)
+ if body := tm.get("/"); !strings.Contains(body, "build") {
+ t.Errorf("expected body to contain \"build\", didn't: %q", body)
}
}
func TestDebugWatcher(t *testing.T) {
- r := newTestRepo()
- r.setStatus("waiting")
- req := httptest.NewRequest("GET", "/debug/watcher/build", nil)
- w := httptest.NewRecorder()
- r.ServeHTTP(w, req)
- if w.Code != 200 {
- t.Fatalf("GET / = code %d, want 200", w.Code)
- }
- body := w.Body.String()
+ tm := newTestMirror(t)
+ tm.commit("hello world")
+ tm.loopOnce()
+
+ body := tm.get("/debug/watcher/build")
if substr := `watcher status for repo: "build"`; !strings.Contains(body, substr) {
t.Fatalf("GET /debug/watcher/build: want %q in body, got %s", substr, body)
}
@@ -72,8 +40,159 @@
}
}
-func mustHaveGit(t *testing.T) {
+func TestArchive(t *testing.T) {
+ tm := newTestMirror(t)
+
+ // Start with a revision we know about.
+ tm.commit("hello world")
+ initialRev := strings.TrimSpace(tm.git(tm.gerrit, "rev-parse", "HEAD"))
+ tm.loopOnce() // fetch the commit.
+ tm.get("/build.tar.gz?rev=" + initialRev)
+
+ // Now test one we don't see yet. It will be fetched automatically.
+ tm.commit("round two")
+ secondRev := strings.TrimSpace(tm.git(tm.gerrit, "rev-parse", "HEAD"))
+ // As of writing, the git version installed on the builders has some kind
+ // of bug that prevents the "git fetch" this triggers from working. Skip.
+ if strings.HasPrefix(tm.git(tm.gerrit, "version"), "git version 2.11") {
+ t.Skip("known-buggy git version")
+ }
+ tm.get("/build.tar.gz?rev=" + secondRev)
+
+ // Pick it up normally and re-fetch it to make sure we don't get confused.
+ tm.loopOnce()
+ tm.get("/build.tar.gz?rev=" + secondRev)
+}
+
+func TestMirror(t *testing.T) {
+ tm := newTestMirror(t)
+ for i := 0; i < 2; i++ {
+ tm.commit(fmt.Sprintf("revision %v", i))
+ rev := tm.git(tm.gerrit, "rev-parse", "HEAD")
+ tm.loopOnce()
+ if githubRev := tm.git(tm.github, "rev-parse", "HEAD"); rev != githubRev {
+ t.Errorf("github HEAD is %v, want %v", githubRev, rev)
+ }
+ if csrRev := tm.git(tm.csr, "rev-parse", "HEAD"); rev != csrRev {
+ t.Errorf("csr HEAD is %v, want %v", csrRev, rev)
+ }
+ }
+}
+
+type testMirror struct {
+ gerrit, github, csr string
+ m *mirror
+ server *httptest.Server
+ buildRepo *repo
+ t *testing.T
+}
+
+// newTestMirror returns a mirror configured to watch the "build" repository
+// and mirror it to GitHub and CSR. All repositories are faked out with local
+// versions created hermetically. The mirror is idle and must be pumped with
+// loopOnce.
+func newTestMirror(t *testing.T) *testMirror {
if _, err := exec.LookPath("git"); err != nil {
t.Skip("skipping; git not in PATH")
}
+
+ tm := &testMirror{
+ gerrit: t.TempDir(),
+ github: t.TempDir(),
+ csr: t.TempDir(),
+ m: &mirror{
+ mux: http.NewServeMux(),
+ cacheDir: t.TempDir(),
+ homeDir: t.TempDir(),
+ repos: map[string]*repo{},
+ },
+ t: t,
+ }
+ tm.m.mux.HandleFunc("/", tm.m.handleRoot)
+ tm.server = httptest.NewServer(tm.m.mux)
+ t.Cleanup(tm.server.Close)
+
+ // Write a git config with each of the relevant repositories replaced by a
+ // local version. The origin is non-bare so we can commit to it; the
+ // destinations are bare so we can push to them.
+ gitConfig := &bytes.Buffer{}
+ overrideRepo := func(fromURL, toDir string, bare bool) {
+ init := exec.Command("git", "init")
+ if bare {
+ init.Args = append(init.Args, "--bare")
+ }
+ init.Dir = toDir
+ if out, err := init.CombinedOutput(); err != nil {
+ t.Fatalf("git init: %v\n%v", err, out)
+ }
+ fmt.Fprintf(gitConfig, "[url %q]\n insteadOf = %v\n", toDir, fromURL)
+ }
+ overrideRepo("https://go.googlesource.com/build", tm.gerrit, false)
+ overrideRepo("git@github.com:golang/build.git", tm.github, true)
+ overrideRepo("https://source.developers.google.com/p/golang-org/r/build", tm.csr, true)
+
+ if err := os.MkdirAll(filepath.Join(tm.m.homeDir, ".config/git"), 0777); err != nil {
+ t.Fatal(err)
+ }
+ if err := ioutil.WriteFile(filepath.Join(tm.m.homeDir, ".config/git/config"), gitConfig.Bytes(), 0777); err != nil {
+ t.Fatal(err)
+ }
+ tm.buildRepo = tm.m.addRepo(&repospkg.Repo{
+ GoGerritProject: "build",
+ ImportPath: "golang.org/x/build",
+ MirrorToGitHub: true,
+ GitHubRepo: "golang/build",
+ MirrorToCSR: true,
+ })
+ if err := tm.buildRepo.init(); err != nil {
+ t.Fatal(err)
+ }
+ if err := tm.m.addMirrors(); err != nil {
+ t.Fatal(err)
+ }
+ return tm
+}
+
+func (tm *testMirror) loopOnce() {
+ tm.t.Helper()
+ if err := tm.buildRepo.loopOnce(); err != nil {
+ tm.t.Fatal(err)
+ }
+}
+
+func (tm *testMirror) commit(content string) {
+ tm.t.Helper()
+ if err := ioutil.WriteFile(filepath.Join(tm.gerrit, "README"), []byte(content), 0777); err != nil {
+ tm.t.Fatal(err)
+ }
+ tm.git(tm.gerrit, "add", ".")
+ tm.git(tm.gerrit, "commit", "-m", content)
+}
+
+func (tm *testMirror) git(dir string, args ...string) string {
+ tm.t.Helper()
+ cmd := exec.Command("git", args...)
+ cmd.Dir = dir
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ tm.t.Fatalf("git: %v, %s", err, out)
+ }
+ return string(out)
+}
+
+func (tm *testMirror) get(path string) string {
+ tm.t.Helper()
+ resp, err := http.Get(tm.server.URL + path)
+ if err != nil {
+ tm.t.Fatal(err)
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ tm.t.Fatal(err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ tm.t.Fatalf("request for %q failed", path)
+ }
+ return string(body)
}
diff --git a/maintner/maintnerd/maintnerd.go b/maintner/maintnerd/maintnerd.go
index 0caaf50..eea394e 100644
--- a/maintner/maintnerd/maintnerd.go
+++ b/maintner/maintnerd/maintnerd.go
@@ -325,7 +325,7 @@
func goGitHubProjects() []string {
var ret []string
for _, r := range repos.ByGerritProject {
- if gr := r.GitHubRepo(); gr != "" {
+ if gr := r.GitHubRepo; gr != "" {
ret = append(ret, gr)
}
}
diff --git a/repos/repos.go b/repos/repos.go
index 5c81b7e..6ab6310 100644
--- a/repos/repos.go
+++ b/repos/repos.go
@@ -36,10 +36,10 @@
// build coordinator knows how to build.
CoordinatorCanBuild bool
- // gitHubRepo is the "org/repo" of where this repo exists on
+ // GitHubRepo is the "org/repo" of where this repo exists on
// GitHub. If MirrorToGitHub is true, this is the
// destination.
- gitHubRepo string
+ GitHubRepo string
// WebsiteDesc is the description of the repo for showing on
// https://golang.org/pkg/#subrepo.
@@ -103,13 +103,13 @@
GoGerritProject: "protobuf",
MirrorToGitHub: true,
ImportPath: "google.golang.org/protobuf",
- gitHubRepo: "protocolbuffers/protobuf-go",
+ GitHubRepo: "protocolbuffers/protobuf-go",
})
add(&Repo{
GoGerritProject: "vscode-go",
MirrorToGitHub: true,
- gitHubRepo: "golang/vscode-go",
+ GitHubRepo: "golang/vscode-go",
})
}
@@ -138,7 +138,7 @@
repo := &Repo{
GoGerritProject: proj,
MirrorToGitHub: true,
- gitHubRepo: "golang/" + proj,
+ GitHubRepo: "golang/" + proj,
}
for _, o := range opts {
o(repo)
@@ -153,7 +153,7 @@
MirrorToGitHub: true,
CoordinatorCanBuild: true,
ImportPath: "golang.org/x/" + proj,
- gitHubRepo: "golang/" + proj,
+ GitHubRepo: "golang/" + proj,
showOnDashboard: true,
}
for _, o := range opts {
@@ -166,7 +166,7 @@
if (r.MirrorToCSR || r.MirrorToGitHub || r.showOnDashboard) && r.GoGerritProject == "" {
panic(fmt.Sprintf("project %+v sets feature(s) that require a GoGerritProject, but has none", r))
}
- if r.MirrorToGitHub && r.gitHubRepo == "" {
+ if r.MirrorToGitHub && r.GitHubRepo == "" {
panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
}
if r.showOnDashboard && !r.CoordinatorCanBuild {
@@ -192,8 +192,3 @@
//
// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }
-
-// GitHubRepo returns the "<org>/<repo>" that this repo either lives
-// at or is mirrored to. It returns the empty string if this repo has no
-// GitHub presence.
-func (r *Repo) GitHubRepo() string { return r.gitHubRepo }
