internal/postgres: insert v1path and series path to paths table
Paths that correlated to a v1path and series path are now also inserted
into the paths table. This will allow us to change units.v1_path to an
integer column with a FK to paths.id and reduce the size of the
idx_units_v1_path index (currently 19 GB in prod).
Additionally, in a future CL, this data can be used to redirect users
when request to pkg.go.dev/<path> 404s. For example:
/github.com/jackc/pgx/pgxpool -> /github.com/jackc/pgx/v4/pgxpool
/github.com/mediocregopher/radix ->
/github.com/mediocregopher/radix/v3 or
/github.com/mediocregopher/radix/v4
Change-Id: I571159ec491eb496326aad8280827b63e0671788
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/280162
Trust: Julie Qiu <julie@golang.org>
Run-TryBot: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
diff --git a/internal/postgres/insert_module.go b/internal/postgres/insert_module.go
index df57736..a69d6c3 100644
--- a/internal/postgres/insert_module.go
+++ b/internal/postgres/insert_module.go
@@ -357,9 +357,15 @@
// Read all existing paths for this module, to avoid a large bulk upsert.
// (We've seen these bulk upserts hang for so long that they time out (10
// minutes)).
- var curPaths []string
+ curPathsSet := map[string]bool{}
for _, u := range m.Units {
- curPaths = append(curPaths, u.Path)
+ curPathsSet[u.Path] = true
+ curPathsSet[internal.V1Path(u.Path, m.ModulePath)] = true
+ curPathsSet[internal.SeriesPathForModule(m.ModulePath)] = true
+ }
+ var curPaths []string
+ for p := range curPathsSet {
+ curPaths = append(curPaths, p)
}
if err := db.RunQuery(ctx, `SELECT id, path FROM paths WHERE path = ANY($1)`,
collect, pq.Array(curPaths)); err != nil {
@@ -367,18 +373,18 @@
}
// Insert any unit paths that we don't already have.
- var pathValues []interface{}
- for _, u := range m.Units {
- if _, ok := pathToID[u.Path]; !ok {
- pathValues = append(pathValues, u.Path)
+ var values []interface{}
+ for _, v := range curPaths {
+ if _, ok := pathToID[v]; !ok {
+ values = append(values, v)
}
}
- if len(pathValues) > 0 {
+ if len(values) > 0 {
// Insert data into the paths table.
pathCols := []string{"path"}
- uniquePathCols := []string{"path"}
returningPathCols := []string{"id", "path"}
- if err := db.BulkUpsertReturning(ctx, "paths", pathCols, pathValues, uniquePathCols, returningPathCols, collect); err != nil {
+ if err := db.BulkInsertReturning(ctx, "paths", pathCols, values,
+ database.OnConflictDoNothing, returningPathCols, collect); err != nil {
return nil, err
}
}
