internal/postgres: repeat search when grouping
When grouping results, search a second time with a larger limit if the
first doesn't return enough results.
Also, remove the Limit search option; it is an implementation detail
that should be determined by DB.Search.
I tested this against 176 distinct queries obtained from the logs over
a 1-hour period, with max results = 10. Although it was somewhat
slower than without grouping, the 99%ile was still under 1 second.
For golang/go#47320
Change-Id: Ie09d6b83bc270a7113abd74b03daf26fa6054b0f
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/336950
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>
TryBot-Result: kokoro <noreply+kokoro@google.com>
diff --git a/internal/postgres/search.go b/internal/postgres/search.go
index 411dd87..cfb5ab4 100644
--- a/internal/postgres/search.go
+++ b/internal/postgres/search.go
@@ -101,8 +101,6 @@
type SearchOptions struct {
// Maximum number of results to return (page size).
MaxResults int
- // Limit for the DB query; defaults to MaxResults.
- Limit int
// Offset for DB query.
Offset int
// Maximum number to use for total result count.
@@ -184,9 +182,30 @@
// the penalty of a deep search that scans nearly every package.
func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error) {
defer derrors.WrapStack(&err, "DB.Search(ctx, %q, %+v)", q, opts)
- if opts.Limit == 0 {
- opts.Limit = opts.MaxResults
+ if experiment.IsActive(ctx, internal.ExperimentSearchGrouping) && !opts.SearchSymbols {
+ const (
+ limitMultiplier1 = 3
+ limitMultiplier2 = 5
+ )
+ // Limit search to more rows than the requested number of results, so
+ // that it can find other packages in the modules it selects.
+ srs, err := db.search(ctx, q, opts, limitMultiplier1*opts.MaxResults)
+ if err != nil {
+ return nil, err
+ }
+ if len(srs) >= opts.MaxResults || numRows(srs) <= limitMultiplier1*opts.MaxResults {
+ return srs, nil
+ }
+ // Grouped search didn't find enough results, but there are more
+ // rows that could potentially match. Try one more time, with a
+ // larger limit.
+ return db.search(ctx, q, opts, limitMultiplier2*opts.MaxResults)
}
+ return db.search(ctx, q, opts, opts.MaxResults)
+}
+
+func (db *DB) search(ctx context.Context, q string, opts SearchOptions, limit int) (_ []*SearchResult, err error) {
+ defer derrors.WrapStack(&err, "search(limit=%d)", limit)
var searchers map[string]searcher
if opts.SearchSymbols &&
@@ -196,7 +215,7 @@
} else {
searchers = pkgSearchers
}
- resp, err := db.hedgedSearch(ctx, q, opts.Limit, opts.Offset, opts.MaxResultCount, searchers, nil)
+ resp, err := db.hedgedSearch(ctx, q, limit, opts.Offset, opts.MaxResultCount, searchers, nil)
if err != nil {
return nil, err
}
@@ -572,6 +591,16 @@
return results
}
+// numRows counts the number of rows in a slice of SearchResults.
+// Grouping will put some rows inside a SearchResult.
+func numRows(rs []*SearchResult) int {
+ n := 0
+ for _, r := range rs {
+ n += 1 + len(r.SameModule)
+ }
+ return n
+}
+
var upsertSearchStatement = fmt.Sprintf(`
INSERT INTO search_documents (
package_path,
