devtools/cmd/seeddb: process versions of the same module sequentially
This time for real.
Before we were only processing the versions expanded from "@all" sequentially,
so if there were two lines in the seed file
M@v1.1.0
M@v1.2.0
they would still be processed concurrently.
Now we handle that case as well.
Change-Id: I6051ff622d68c316386e7c09de6ddf603c8ab273
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/341349
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/devtools/ci/ci.sh b/devtools/ci/ci.sh
index 929b786..ae86c26 100755
--- a/devtools/ci/ci.sh
+++ b/devtools/ci/ci.sh
@@ -102,7 +102,6 @@
echo "----------------------------------------"
echo "Running e2e tests"
echo "----------------------------------------"
-echo "Temporarily disabled until seeddb is fixed."
./tests/e2e/run.sh
print_duration_and_reset
diff --git a/devtools/cmd/seeddb/main.go b/devtools/cmd/seeddb/main.go
index f04355a..1b391a3 100644
--- a/devtools/cmd/seeddb/main.go
+++ b/devtools/cmd/seeddb/main.go
@@ -78,6 +78,16 @@
return err
}
+ // Expand versions and group by module path.
+ versionsByPath := map[string][]string{}
+ for _, m := range seedModules {
+ vers, err := versions(ctx, proxyClient, m)
+ if err != nil {
+ return err
+ }
+ versionsByPath[m.Path] = append(versionsByPath[m.Path], vers...)
+ }
+
r := results{}
g := new(errgroup.Group)
f := &worker.Fetcher{
@@ -85,16 +95,13 @@
SourceClient: sourceClient,
DB: postgres.New(db),
}
- for _, m := range seedModules {
- m := m
- vers, err := versions(ctx, proxyClient, m)
- if err != nil {
- return err
- }
+ for path, vers := range versionsByPath {
+ path := path
+ vers := vers
// Process versions of the same module sequentially, to avoid DB contention.
g.Go(func() error {
for _, v := range vers {
- if err := fetch(ctx, db, f, m.Path, v, &r); err != nil {
+ if err := fetch(ctx, db, f, path, v, &r); err != nil {
return err
}
}
