maintner/maintnerd: add --config=devgo to sync from prod data on start-up
This makes maintnerd development easier. The new "devgo" config is
like the production "go" config, but syncs the production data to
local disk before starting, so there's no crazy storm of sync activity
at the beginning.
Change-Id: I7602d7327878683f26e6a5e94c617c01143fbd67
Reviewed-on: https://go-review.googlesource.com/43615
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/maintner/godata/godata.go b/maintner/godata/godata.go
index cdcdffb..559b71e 100644
--- a/maintner/godata/godata.go
+++ b/maintner/godata/godata.go
@@ -39,7 +39,7 @@
// See https://godoc.org/golang.org/x/build/maintner#Corpus for how
// to walk the data structure. Enjoy.
func Get(ctx context.Context) (*maintner.Corpus, error) {
- targetDir := filepath.Join(xdgCacheDir(), "golang-maintner")
+ targetDir := Dir()
if err := os.MkdirAll(targetDir, 0700); err != nil {
return nil, err
}
@@ -51,6 +51,11 @@
return corpus, nil
}
+// Dir returns the directory containing the cached mutation logs.
+func Dir() string {
+ return filepath.Join(xdgCacheDir(), "golang-maintner")
+}
+
// xdgCacheDir returns the XDG Base Directory Specification cache
// directory.
func xdgCacheDir() string {
diff --git a/maintner/maintnerd/maintnerd.go b/maintner/maintnerd/maintnerd.go
index 500e22a..d41e67c 100644
--- a/maintner/maintnerd/maintnerd.go
+++ b/maintner/maintnerd/maintnerd.go
@@ -48,7 +48,7 @@
watchGithub = flag.String("watch-github", "", "Comma-separated list of owner/repo pairs to slurp")
watchGerrit = flag.String("watch-gerrit", "", `Comma-separated list of Gerrit projects to watch, each of form "hostname/project" (e.g. "go.googlesource.com/go")`)
pubsub = flag.String("pubsub", "", "If non-empty, the golang.org/x/build/cmd/pubsubhelper URL scheme and hostname, without path")
- config = flag.String("config", "", "If non-empty, the name of a pre-defined config. Valid options are 'go' to be the primary Go server; 'godata' to run the server locally using the godata package.")
+ config = flag.String("config", "", "If non-empty, the name of a pre-defined config. Valid options are 'go' to be the primary Go server; 'godata' to run the server locally using the godata package, and 'devgo' to act like 'go', but mirror from godata at start-up.")
dataDir = flag.String("data-dir", "", "Local directory to write protobuf files to (default $HOME/var/maintnerd)")
debug = flag.Bool("debug", false, "Print debug logging information")
@@ -96,6 +96,24 @@
switch *config {
case "":
// Nothing
+ case "devgo":
+ dir := godata.Dir()
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("Syncing from https://maintner.golang.org/logs to %s", dir)
+ mutSrc := maintner.NewNetworkMutationSource("https://maintner.golang.org/logs", dir)
+ for evt := range mutSrc.GetMutations(context.Background()) {
+ if evt.Err != nil {
+ log.Fatal(evt.Err)
+ }
+ if evt.End {
+ break
+ }
+ }
+ syncProdToDevMutationLogs()
+ log.Printf("Synced from https://maintner.golang.org/logs.")
+ setGoConfig()
case "go":
setGoConfig()
case "godata":
@@ -365,3 +383,57 @@
tc.SetKeepAlivePeriod(3 * time.Minute)
return tc, nil
}
+
+func syncProdToDevMutationLogs() {
+ src := godata.Dir()
+ dst := *dataDir
+
+ want := map[string]int64{} // basename => size
+
+ srcFis, err := ioutil.ReadDir(src)
+ if err != nil {
+ log.Fatal(err)
+ }
+ dstFis, err := ioutil.ReadDir(dst)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ for _, fi := range srcFis {
+ name := fi.Name()
+ if !strings.HasSuffix(name, ".mutlog") {
+ continue
+ }
+ // The DiskMutationLogger (as we'l use in the dst dir)
+ // prepends "maintner-". So prepend that here ahead
+ // of time, even though the network mutation source's
+ // cache doesn't.
+ want["maintner-"+name] = fi.Size()
+ }
+
+ for _, fi := range dstFis {
+ name := fi.Name()
+ if !strings.HasSuffix(name, ".mutlog") {
+ continue
+ }
+ if want[name] == fi.Size() {
+ delete(want, name)
+ continue
+ }
+ log.Printf("dst file %q unwanted", name)
+ if err := os.Remove(filepath.Join(dst, name)); err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ for name := range want {
+ log.Printf("syncing %s from %s to %s", name, src, dst)
+ slurp, err := ioutil.ReadFile(filepath.Join(src, strings.TrimPrefix(name, "maintner-")))
+ if err != nil {
+ log.Fatal(err)
+ }
+ if err := ioutil.WriteFile(filepath.Join(dst, name), slurp, 0644); err != nil {
+ log.Fatal(err)
+ }
+ }
+}
