cmd/gitmirror: add CSR support
Add support for CSR writes, disabled for now and basically untested.
This relies on modifying the global git config, so rather than using the
user's real home directory, use a temp dir instead. This also sets us up
for testing.
Also, write remotes into $GIT_DIR/remotes rather than the config file.
Much easier.
Change-Id: If45a468ad715ee24660d927b832096a70a0ffd4f
Reviewed-on: https://go-review.googlesource.com/c/build/+/324398
Trust: Heschi Kreinick <heschi@google.com>
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/cmd/gitmirror/gitmirror.go b/cmd/gitmirror/gitmirror.go
index 62315c5..38f3b8d 100644
--- a/cmd/gitmirror/gitmirror.go
+++ b/cmd/gitmirror/gitmirror.go
@@ -18,6 +18,7 @@
"net/http"
"os"
"os/exec"
+ "os/signal"
"path/filepath"
"runtime"
"sort"
@@ -66,18 +67,24 @@
if err != nil {
log.Fatalf("creating cache dir: %v", err)
}
+ credsDir, err := ioutil.TempDir("", "gitmirror-credentials")
+ if err != nil {
+ log.Fatalf("creating credentials dir: %v", err)
+ }
+ defer os.RemoveAll(credsDir)
m := &mirror{
mux: http.DefaultServeMux,
repos: map[string]*repo{},
cacheDir: cacheDir,
+ homeDir: credsDir,
gerritClient: gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth),
}
http.HandleFunc("/", m.handleRoot)
var eg errgroup.Group
- for name := range repospkg.ByGerritProject {
- r := m.addRepo(name)
+ for _, repo := range repospkg.ByGerritProject {
+ r := m.addRepo(repo)
eg.Go(r.init)
}
if err := eg.Wait(); err != nil {
@@ -85,11 +92,11 @@
}
if *flagMirror {
- if err := writeCredentials(); err != nil {
- log.Fatalf("writing ssh credentials: %v", err)
+ if err := writeCredentials(credsDir); err != nil {
+ log.Fatalf("writing git credentials: %v", err)
}
- if err := m.runMirrors(); err != nil {
- log.Fatalf("running mirror: %v", err)
+ if err := m.addMirrors(); err != nil {
+ log.Fatalf("configuring mirrors: %v", err)
}
}
@@ -99,38 +106,58 @@
go m.pollGerritAndTickle()
go m.subscribeToMaintnerAndTickleLoop()
- select {}
+ shutdown := make(chan os.Signal, 1)
+ signal.Notify(shutdown, os.Interrupt)
+ <-shutdown
}
-func writeCredentials() error {
+func writeCredentials(home string) error {
sc := secret.MustNewClient()
defer sc.Close()
- home, err := os.UserHomeDir()
- if err != nil {
- return err
- }
- sshDir := filepath.Join(home, ".ssh")
- sshKey := filepath.Join(sshDir, "id_ed25519")
- if _, err := os.Stat(sshKey); err == nil {
- log.Printf("Using github ssh key at %v", sshKey)
- return nil
- }
-
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
+ sshConfig := &bytes.Buffer{}
+ gitConfig := &bytes.Buffer{}
+ sshConfigPath := filepath.Join(home, "ssh_config")
+ // ssh ignores $HOME in favor of /etc/passwd, so we need to override ssh_config explicitly.
+ fmt.Fprintf(gitConfig, "[core]\n sshCommand=\"ssh -F %v\"\n", sshConfigPath)
+
+ // GitHub key, used as the default SSH private key.
privKey, err := sc.Retrieve(ctx, secret.NameGitHubSSHKey)
- if err != nil || len(privKey) == 0 {
- return fmt.Errorf("can't mirror to github without %q GCP secret manager or file %v", secret.NameGitHubSSHKey, sshKey)
+ if err != nil {
+ return fmt.Errorf("reading github key from secret manager: %v", err)
}
- if err := os.MkdirAll(sshDir, 0700); err != nil {
+ privKeyPath := filepath.Join(home, secret.NameGitHubSSHKey)
+ if err := ioutil.WriteFile(privKeyPath, []byte(privKey+"\n"), 0600); err != nil {
return err
}
- if err := ioutil.WriteFile(sshKey, []byte(privKey+"\n"), 0600); err != nil {
+ fmt.Fprintf(sshConfig, "Host github.com\n IdentityFile %v\n", privKeyPath)
+
+ // gitmirror service key, used to gcloud auth for CSR writes.
+ serviceKey, err := sc.Retrieve(ctx, secret.NameGitMirrorServiceKey)
+ if err != nil {
+ return fmt.Errorf("reading service key from secret manager: %v", err)
+ }
+ serviceKeyPath := filepath.Join(home, secret.NameGitMirrorServiceKey)
+ if err := ioutil.WriteFile(serviceKeyPath, []byte(serviceKey), 0600); err != nil {
return err
}
- log.Printf("Wrote %s from GCP secret manager.", sshKey)
+ gcloud := exec.CommandContext(ctx, "gcloud", "auth", "activate-service-account", "--key-file", serviceKeyPath)
+ gcloud.Env = append(os.Environ(), "HOME="+home)
+ if out, err := gcloud.CombinedOutput(); err != nil {
+ return fmt.Errorf("gcloud auth failed: %v\n%v", err, out)
+ }
+ fmt.Fprintf(gitConfig, "[credential \"https://source.developers.google.com\"]\n helper=gcloud.sh\n")
+
+ if err := ioutil.WriteFile(filepath.Join(home, ".gitconfig"), gitConfig.Bytes(), 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(sshConfigPath, sshConfig.Bytes(), 0600); err != nil {
+ return err
+ }
+
return nil
}
@@ -163,16 +190,20 @@
// A mirror watches Gerrit repositories, fetching the latest commits and
// optionally mirroring them.
type mirror struct {
- mux *http.ServeMux
- repos map[string]*repo
- cacheDir string
+ mux *http.ServeMux
+ repos map[string]*repo
+ cacheDir string
+ // homeDir is used as $HOME for all commands, allowing easy configuration overrides.
+ homeDir string
gerritClient *gerrit.Client
}
-func (m *mirror) addRepo(name string) *repo {
+func (m *mirror) addRepo(meta *repospkg.Repo) *repo {
+ name := meta.GoGerritProject
r := &repo{
name: name,
url: goBase + name,
+ meta: meta,
root: filepath.Join(m.cacheDir, name),
changed: make(chan bool, 1),
mirror: m,
@@ -183,23 +214,18 @@
return r
}
-// runMirrors sets up and starts mirroring for the repositories that are
-// configured to be mirrored.
-func (m *mirror) runMirrors() error {
- for name, repo := range m.repos {
- meta, ok := repospkg.ByGerritProject[name]
- if !ok || !meta.MirrorToGitHub {
- continue
+// addMirrors sets up mirroring for repositories that need it.
+func (m *mirror) addMirrors() error {
+ for _, repo := range m.repos {
+ if repo.meta.MirrorToGitHub {
+ if err := repo.addRemote("github", "git@github.com:"+repo.meta.GitHubRepo()+".git"); err != nil {
+ return fmt.Errorf("adding GitHub remote: %v", err)
+ }
}
- if err := repo.addRemote("github", "git@github.com:"+meta.GitHubRepo()+".git",
- // We want to include only the refs/heads/* and refs/tags/* namespaces
- // in the mirrors. They correspond to published branches and tags.
- // Leave out internal Gerrit namespaces such as refs/changes/*,
- // refs/users/*, etc., because they're not helpful on other hosts.
- "push = +refs/heads/*:refs/heads/*",
- "push = +refs/tags/*:refs/tags/*",
- ); err != nil {
- return fmt.Errorf("adding remote: %v", err)
+ if repo.meta.MirrorToCSR {
+ if err := repo.addRemote("csr", "https://source.developers.google.com/p/golang-org/r/"+repo.name); err != nil {
+ return fmt.Errorf("adding CSR remote: %v", err)
+ }
}
}
return nil
@@ -271,7 +297,8 @@
type repo struct {
name string
url string
- root string // on-disk location of the git repo, *cacheDir/name
+ root string // on-disk location of the git repo, *cacheDir/name
+ meta *repospkg.Repo
changed chan bool // sent to when a change comes in
status statusRing
dests []string // destination remotes to mirror to
@@ -331,6 +358,7 @@
stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Dir = r.root
+ cmd.Env = append(os.Environ(), "HOME="+r.mirror.homeDir)
cmd.Stdout, cmd.Stderr = stdout, stderr
err := cmd.Run()
return stdout.Bytes(), stderr.Bytes(), err
@@ -386,32 +414,19 @@
r.status.add(status)
}
-func (r *repo) addRemote(name, url string, opts ...string) error {
+func (r *repo) addRemote(name, url string) error {
r.dests = append(r.dests, name)
- if _, _, err := r.runGitQuiet("remote", "remove", name); err != nil {
- // Exit status 2 means not found, which is fine.
- if ee, ok := err.(*exec.ExitError); !ok || ee.ExitCode() != 2 {
- return err
- }
- }
- gitConfig := filepath.Join(r.root, "config")
- f, err := os.OpenFile(gitConfig, os.O_WRONLY|os.O_APPEND, os.ModePerm)
- if err != nil {
+ if err := os.MkdirAll(filepath.Join(r.root, "remotes"), 0777); err != nil {
return err
}
- _, err = fmt.Fprintf(f, "\n[remote %q]\n\turl = %v\n", name, url)
- if err != nil {
- f.Close()
- return err
- }
- for _, o := range opts {
- _, err := fmt.Fprintf(f, "\t%s\n", o)
- if err != nil {
- f.Close()
- return err
- }
- }
- return f.Close()
+ // We want to include only the refs/heads/* and refs/tags/* namespaces
+ // in the mirrors. They correspond to published branches and tags.
+ // Leave out internal Gerrit namespaces such as refs/changes/*,
+ // refs/users/*, etc., because they're not helpful on other hosts.
+ remote := "URL: " + url + "\n" +
+ "Push: +refs/heads/*:refs/heads/*\n" +
+ "Push: +refs/tags/*:refs/tags/*\n"
+ return ioutil.WriteFile(filepath.Join(r.root, "remotes", name), []byte(remote), 0777)
}
// Loop continuously runs "git fetch" in the repo, checks for new
diff --git a/internal/secret/gcp_secret_manager.go b/internal/secret/gcp_secret_manager.go
index da02dff..5d1ec82 100644
--- a/internal/secret/gcp_secret_manager.go
+++ b/internal/secret/gcp_secret_manager.go
@@ -34,6 +34,9 @@
// NameGitHubSSHKey is the secret name for the GitHub SSH private key.
NameGitHubSSHKey = "github-ssh-private-key"
+ // NameGitMirrorServiceKey is the secret name for the gitmirror service account key.
+ NameGitMirrorServiceKey = "gitmirror-service-key"
+
// NameGobotPassword is the secret name for the Gobot password.
NameGobotPassword = "gobot-password"
diff --git a/repos/repos.go b/repos/repos.go
index a8aa2c3..5c81b7e 100644
--- a/repos/repos.go
+++ b/repos/repos.go
@@ -22,6 +22,12 @@
// gitHubRepo must both be defined.
MirrorToGitHub bool
+ // MirrorToCSR controls whether this repo is mirrored from
+ // Gerrit to Cloud Source Repositories. If true, GoGerritProject
+ // must be defined. It will be mirrored to a CSR repo with the
+ // same name as the Gerrit repo.
+ MirrorToCSR bool
+
// showOnDashboard is whether to show the repo on the bottom
// of build.golang.org in the repo overview section.
showOnDashboard bool
@@ -157,21 +163,14 @@
}
func add(r *Repo) {
- if r.MirrorToGitHub {
- if r.gitHubRepo == "" {
- panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
- }
- if r.GoGerritProject == "" {
- panic(fmt.Sprintf("project %+v has MirrorToGitHub but no GoGerritProject", r))
- }
+ if (r.MirrorToCSR || r.MirrorToGitHub || r.showOnDashboard) && r.GoGerritProject == "" {
+ panic(fmt.Sprintf("project %+v sets feature(s) that require a GoGerritProject, but has none", r))
}
- if r.showOnDashboard {
- if !r.CoordinatorCanBuild {
- panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
- }
- if r.GoGerritProject == "" {
- panic(fmt.Sprintf("project %+v is showOnDashboard but has no Gerrit project", r))
- }
+ if r.MirrorToGitHub && r.gitHubRepo == "" {
+ panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
+ }
+ if r.showOnDashboard && !r.CoordinatorCanBuild {
+ panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
}
if p := r.GoGerritProject; p != "" {
