cmd/pkgsite: add multi-module support, invalidation, and search
This change adds a new goPackagesModuleGetter, which uses
x/tools/go/packages to query package information for modules requested
by the pkgsite command.
Additionally, add new extension interfaces that allow getters to add
support for search and content invalidation. The go/packages getter uses
these extensions to implement search (via a simple fuzzy-matching
algorithm copied from x/tools), and invalidation via statting package
files.
Along the way, refactor slightly for testing ergonomics.
Updates golang/go#40371
Updates golang/go#50229
Fixes golang/go#54479
Change-Id: Iea91a4d6327707733cbbc4f74a9d93052f33e295
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/474295
TryBot-Result: kokoro <noreply+kokoro@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index 29fd67e..127d35e 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -8,14 +8,26 @@
//
// To install, run `go install ./cmd/pkgsite` from the pkgsite repo root.
//
-// With no arguments, pkgsite will serve docs for the module in the current
-// directory, which must have a go.mod file:
+// With no arguments, pkgsite will serve docs for main modules relative to the
+// current directory, i.e. the modules listed by `go list -m`. This is
+// typically the module defined by the nearest go.mod file in a parent
+// directory. However, this may include multiple main modules when using a
+// go.work file to define a [workspace].
//
-// cd ~/repos/cue && pkgsite
+// For example, both of the following the following forms could be used to work
+// on the module defined in repos/cue/go.mod:
//
-// This form will also serve all of the module's required modules at their
-// required versions. You can disable serving the required modules by passing
-// -list=false.
+// The single module form:
+//
+// cd repos/cue && pkgsite
+//
+// The multiple module form:
+//
+// go work init repos/cue repos/other && pkgsite
+//
+// By default, the resulting server will also serve all of the module's
+// dependencies at their required versions. You can disable serving the
+// required modules by passing -list=false.
//
// You can also serve docs from your module cache, directly from the proxy
// (it uses the GOPROXY environment variable), or both:
@@ -32,13 +44,12 @@
// while to appear the first time because the Go repo must be cloned and
// processed. If you clone the repo yourself (https://go.googlesource.com/go),
// you can provide its location with the -gorepo flag to save a little time.
+//
+// [workspace]: https://go.dev/ref/mod#workspaces
package main
import (
- "bytes"
"context"
- "encoding/json"
- "errors"
"flag"
"fmt"
"net/http"
@@ -65,16 +76,30 @@
const defaultAddr = "localhost:8080" // default webserver address
var (
- gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
- httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
- useCache = flag.Bool("cache", false, "fetch from the module cache")
- cacheDir = flag.String("cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
- useProxy = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
- goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem")
- useListedMods = flag.Bool("list", true, "for each path, serve all modules in build list")
+ httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
+ goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem")
+ useProxy = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
+ // other flags are bound to serverConfig below
)
+type serverConfig struct {
+ paths []string
+ gopathMode bool
+ useCache bool
+ cacheDir string
+ useListedMods bool
+
+ proxy *proxy.Client // client, or nil; controlled by the -proxy flag
+}
+
func main() {
+ var serverCfg serverConfig
+
+ flag.BoolVar(&serverCfg.gopathMode, "gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
+ flag.BoolVar(&serverCfg.useCache, "cache", false, "fetch from the module cache")
+ flag.StringVar(&serverCfg.cacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
+ flag.BoolVar(&serverCfg.useListedMods, "list", true, "for each path, serve all modules in build list")
+
flag.Usage = func() {
out := flag.CommandLine.Output()
fmt.Fprintf(out, "usage: %s [flags] [PATHS ...]\n", os.Args[0])
@@ -83,40 +108,19 @@
flag.PrintDefaults()
}
flag.Parse()
- ctx := context.Background()
+ serverCfg.paths = collectPaths(flag.Args())
- paths := collectPaths(flag.Args())
- if len(paths) == 0 && !*useCache && !*useProxy {
- paths = []string{"."}
- }
-
- var modCacheDir string
- if *useCache || *useListedMods {
- modCacheDir = *cacheDir
- if modCacheDir == "" {
- var err error
- modCacheDir, err = defaultCacheDir()
- if err != nil {
- die("%v", err)
- }
- if modCacheDir == "" {
- die("empty value for GOMODCACHE")
- }
- }
- }
-
- if *useCache || *useProxy {
+ if serverCfg.useCache || *useProxy {
fmt.Fprintf(os.Stderr, "BYPASSING LICENSE CHECKING: MAY DISPLAY NON-REDISTRIBUTABLE INFORMATION\n")
}
- var prox *proxy.Client
if *useProxy {
url := os.Getenv("GOPROXY")
if url == "" {
die("GOPROXY environment variable is not set")
}
var err error
- prox, err = proxy.New(url)
+ serverCfg.proxy, err = proxy.New(url)
if err != nil {
die("connecting to proxy: %s", err)
}
@@ -126,23 +130,12 @@
stdlib.SetGoRepoPath(*goRepoPath)
}
- var cacheMods []internal.Modver
- if *useListedMods && !*useCache {
- var err error
- paths, cacheMods, err = listModsForPaths(paths, modCacheDir)
- if err != nil {
- die("listing mods (consider passing -list=false): %v", err)
- }
+ ctx := context.Background()
+ server, err := buildServer(ctx, serverCfg)
+ if err != nil {
+ die(err.Error())
}
- getters, err := buildGetters(ctx, paths, *gopathMode, modCacheDir, cacheMods, prox)
- if err != nil {
- die("%s", err)
- }
- server, err := newServer(getters, prox)
- if err != nil {
- die("%s", err)
- }
router := http.NewServeMux()
server.Install(router.Handle, nil, nil)
mw := middleware.Timeout(54 * time.Second)
@@ -150,6 +143,59 @@
die("%v", http.ListenAndServe(*httpAddr, mw(router)))
}
+func die(format string, args ...any) {
+ fmt.Fprintf(os.Stderr, format, args...)
+ fmt.Fprintln(os.Stderr)
+ os.Exit(1)
+}
+
+func buildServer(ctx context.Context, serverCfg serverConfig) (*frontend.Server, error) {
+ if len(serverCfg.paths) == 0 && !serverCfg.useCache && serverCfg.proxy == nil {
+ serverCfg.paths = []string{"."}
+ }
+
+ cfg := getterConfig{
+ dirs: serverCfg.paths,
+ proxy: serverCfg.proxy,
+ }
+
+ // By default, the requested paths are interpreted as directories. However,
+ // if -gopath_mode is set, they are interpreted as relative paths to modules
+ // in a GOPATH directory.
+ if serverCfg.gopathMode {
+ var err error
+ cfg.dirs, err = getGOPATHModuleDirs(ctx, serverCfg.paths)
+ if err != nil {
+ return nil, fmt.Errorf("searching GOPATH: %v", err)
+ }
+ }
+
+ cfg.pattern = "./..."
+ if serverCfg.useListedMods {
+ cfg.pattern = "all"
+ }
+
+ if serverCfg.useCache {
+ cfg.modCacheDir = serverCfg.cacheDir
+ if cfg.modCacheDir == "" {
+ var err error
+ cfg.modCacheDir, err = defaultCacheDir()
+ if err != nil {
+ return nil, err
+ }
+ if cfg.modCacheDir == "" {
+ return nil, fmt.Errorf("empty value for GOMODCACHE")
+ }
+ }
+ }
+
+ getters, err := buildGetters(ctx, cfg)
+ if err != nil {
+ return nil, err
+ }
+ return newServer(getters, cfg.proxy)
+}
+
func collectPaths(args []string) []string {
var paths []string
for _, arg := range args {
@@ -158,48 +204,93 @@
return paths
}
-func buildGetters(ctx context.Context, paths []string, gopathMode bool, downloadDir string, cacheMods []internal.Modver, prox *proxy.Client) ([]fetch.ModuleGetter, error) {
- getters := buildPathGetters(ctx, paths, gopathMode)
- if downloadDir != "" {
- g, err := fetch.NewFSProxyModuleGetter(downloadDir, cacheMods)
+// getGOPATHModuleDirs returns the absolute paths to directories in GOPATH
+// corresponding to the requested module paths.
+//
+// An error is returned if any operations failed unexpectedly. If individual
+// module paths are not found, an error is logged and the path skipped. An
+// error is returned only if no module paths resolved to a GOPATH directory.
+func getGOPATHModuleDirs(ctx context.Context, modulePaths []string) ([]string, error) {
+ gopath, err := runGo("", "env", "GOPATH")
+ if err != nil {
+ return nil, err
+ }
+ gopaths := filepath.SplitList(string(gopath))
+
+ var dirs []string
+ for _, path := range modulePaths {
+ dir := ""
+ for _, gopath := range gopaths {
+ candidate := filepath.Join(gopath, "src", path)
+ info, err := os.Stat(candidate)
+ if err == nil && info.IsDir() {
+ dir = candidate
+ break
+ }
+ if err != nil && !os.IsNotExist(err) {
+ return nil, err
+ }
+ }
+ if dir == "" {
+ log.Errorf(ctx, "ERROR: no GOPATH directory contains %q", path)
+ } else {
+ dirs = append(dirs, dir)
+ }
+ }
+
+ if len(modulePaths) > 0 && len(dirs) == 0 {
+ return nil, fmt.Errorf("no GOPATH directories contain any of the requested module(s)")
+ }
+ return dirs, nil
+}
+
+// getterConfig defines the set of getters for the server to use.
+// See buildGetters.
+type getterConfig struct {
+ dirs []string // local directories to serve
+ pattern string // go/packages query to load in each directory
+ modCacheDir string // path to module cache, or ""
+ proxy *proxy.Client // proxy client, or nil
+}
+
+// buildGetters constructs module getters based on the given configuration.
+//
+// Getters are returned in the following priority order:
+// 1. local getters for cfg.dirs, in the given order
+// 2. a module cache getter, if cfg.modCacheDir != ""
+// 3. a proxy getter, if cfg.proxy != nil
+func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter, error) {
+ var getters []fetch.ModuleGetter
+
+ // Load local getters for each directory.
+ for _, dir := range cfg.dirs {
+ mg, err := fetch.NewGoPackagesModuleGetter(ctx, dir, cfg.pattern)
+ if err != nil {
+ log.Errorf(ctx, "Loading packages from %s: %v", dir, err)
+ } else {
+ getters = append(getters, mg)
+ }
+ }
+ if len(getters) == 0 && len(cfg.dirs) > 0 {
+ return nil, fmt.Errorf("failed to load any module(s) at %v", cfg.dirs)
+ }
+
+ // Add a getter for the local module cache.
+ if cfg.modCacheDir != "" {
+ g, err := fetch.NewModCacheGetter(cfg.modCacheDir)
if err != nil {
return nil, err
}
getters = append(getters, g)
}
- if prox != nil {
- getters = append(getters, fetch.NewProxyModuleGetter(prox, source.NewClient(time.Second)))
+
+ // Add a proxy
+ if cfg.proxy != nil {
+ getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(time.Second)))
}
return getters, nil
}
-func buildPathGetters(ctx context.Context, paths []string, gopathMode bool) []fetch.ModuleGetter {
- var getters []fetch.ModuleGetter
- loaded := len(paths)
- for _, path := range paths {
- var (
- mg fetch.ModuleGetter
- err error
- )
- if gopathMode {
- mg, err = fetchdatasource.NewGOPATHModuleGetter(path)
- } else {
- mg, err = fetch.NewDirectoryModuleGetter("", path)
- }
- if err != nil {
- log.Error(ctx, err)
- loaded--
- } else {
- getters = append(getters, mg)
- }
- }
-
- if loaded == 0 && len(paths) > 0 {
- die("failed to load module(s) at %v", paths)
- }
- return getters
-}
-
func newServer(getters []fetch.ModuleGetter, prox *proxy.Client) (*frontend.Server, error) {
lds := fetchdatasource.Options{
Getters: getters,
@@ -232,32 +323,6 @@
return strings.TrimSpace(string(out)), nil
}
-// listedMod has a subset of the fields written by `go list -m`.
-type listedMod struct {
- internal.Modver
- GoMod string // absolute path to go.mod file; in download cache or replaced
- Indirect bool
-}
-
-var listModules = _listModules
-
-func _listModules(dir string) ([]listedMod, error) {
- out, err := runGo(dir, "list", "-json", "-m", "-mod", "readonly", "all")
- if err != nil {
- return nil, err
- }
- d := json.NewDecoder(bytes.NewReader(out))
- var ms []listedMod
- for d.More() {
- var m listedMod
- if err := d.Decode(&m); err != nil {
- return nil, err
- }
- ms = append(ms, m)
- }
- return ms, nil
-}
-
func runGo(dir string, args ...string) ([]byte, error) {
cmd := exec.Command("go", args...)
cmd.Dir = dir
@@ -267,35 +332,3 @@
}
return out, nil
}
-
-func listModsForPaths(paths []string, cacheDir string) ([]string, []internal.Modver, error) {
- var outPaths []string
- var cacheMods []internal.Modver
- for _, p := range paths {
- lms, err := listModules(p)
- if err != nil {
- return nil, nil, err
- }
- for _, lm := range lms {
- // Ignore indirect modules.
- if lm.Indirect {
- continue
- }
- if lm.GoMod == "" {
- return nil, nil, errors.New("empty GoMod: please file a pkgsite bug at https://go.dev/issues/new")
- }
- if strings.HasPrefix(lm.GoMod, cacheDir) {
- cacheMods = append(cacheMods, lm.Modver)
- } else { // probably the result of a replace directive
- outPaths = append(outPaths, filepath.Dir(lm.GoMod))
- }
- }
- }
- return outPaths, cacheMods, nil
-}
-
-func die(format string, args ...any) {
- fmt.Fprintf(os.Stderr, format, args...)
- fmt.Fprintln(os.Stderr)
- os.Exit(1)
-}
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index c78fda7..50b3164 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -16,10 +16,10 @@
"github.com/google/go-cmp/cmp"
"golang.org/x/net/html"
- "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/proxy/proxytest"
"golang.org/x/pkgsite/internal/testing/htmlcheck"
+ "golang.org/x/pkgsite/internal/testing/testhelper"
)
var (
@@ -51,20 +51,19 @@
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()
- localGetter := "Dir(example.com/testmod, " + abs(localModule) + ")"
+ localGetter := "Dir(" + abs(localModule) + ")"
cacheGetter := "FSProxy(" + abs(cacheDir) + ")"
for _, test := range []struct {
name string
- paths []string
- cmods []internal.Modver
+ dirs []string
cacheDir string
- prox *proxy.Client
+ proxy *proxy.Client
want []string
}{
{
- name: "local only",
- paths: []string{localModule},
- want: []string{localGetter},
+ name: "local only",
+ dirs: []string{localModule},
+ want: []string{localGetter},
},
{
name: "cache",
@@ -72,27 +71,25 @@
want: []string{cacheGetter},
},
{
- name: "proxy",
- prox: prox,
- want: []string{"Proxy"},
+ name: "proxy",
+ proxy: prox,
+ want: []string{"Proxy"},
},
{
name: "all three",
- paths: []string{localModule},
+ dirs: []string{localModule},
cacheDir: cacheDir,
- prox: prox,
+ proxy: prox,
want: []string{localGetter, cacheGetter, "Proxy"},
},
- {
- name: "list",
- paths: []string{localModule},
- cacheDir: cacheDir,
- cmods: []internal.Modver{{Path: "foo", Version: "v1.2.3"}},
- want: []string{localGetter, "FSProxy(" + abs(cacheDir) + ", foo@v1.2.3)"},
- },
} {
t.Run(test.name, func(t *testing.T) {
- getters, err := buildGetters(ctx, test.paths, false, test.cacheDir, test.cmods, test.prox)
+ getters, err := buildGetters(ctx, getterConfig{
+ dirs: test.dirs,
+ pattern: "./...",
+ modCacheDir: test.cacheDir,
+ proxy: test.proxy,
+ })
if err != nil {
t.Fatal(err)
}
@@ -119,34 +116,40 @@
return a
}
- localModule := repoPath("internal/fetch/testdata/has_go_mod")
+ localModule, _ := testhelper.WriteTxtarToTempDir(t, `
+-- go.mod --
+module example.com/testmod
+-- a.go --
+package a
+`)
cacheDir := repoPath("internal/fetch/testdata/modcache")
testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()
- getters, err := buildGetters(context.Background(), []string{localModule}, false, cacheDir, nil, prox)
- if err != nil {
- t.Fatal(err)
+ defaultConfig := serverConfig{
+ paths: []string{localModule},
+ gopathMode: false,
+ useListedMods: true,
+ useCache: true,
+ cacheDir: cacheDir,
+ proxy: prox,
}
- server, err := newServer(getters, prox)
- if err != nil {
- t.Fatal(err)
- }
- mux := http.NewServeMux()
- server.Install(mux.Handle, nil, nil)
modcacheChecker := in("",
in(".Documentation", hasText("var V = 1")),
sourceLinks(path.Join(abs(cacheDir), "modcache.com@v1.0.0"), "a.go"))
+ ctx := context.Background()
for _, test := range []struct {
name string
+ cfg serverConfig
url string
want htmlcheck.Checker
}{
{
"local",
+ defaultConfig,
"example.com/testmod",
in("",
in(".Documentation", hasText("There is no documentation for this package.")),
@@ -154,21 +157,50 @@
},
{
"modcache",
+ defaultConfig,
"modcache.com@v1.0.0",
modcacheChecker,
},
{
"modcache latest",
+ defaultConfig,
"modcache.com",
modcacheChecker,
},
{
"proxy",
+ defaultConfig,
"example.com/single/pkg",
hasText("G is new in v1.1.0"),
},
+ {
+ "search",
+ defaultConfig,
+ "search?q=a",
+ in(".SearchResults",
+ hasText("example.com/testmod"),
+ ),
+ },
+ {
+ "search",
+ defaultConfig,
+ "search?q=zzz",
+ in(".SearchResults",
+ hasText("no matches"),
+ ),
+ },
+ // TODO(rfindley): add more tests, including a test for the standard
+ // library once it doesn't go through the stdlib package.
+ // See also golang/go#58923.
} {
t.Run(test.name, func(t *testing.T) {
+ server, err := buildServer(ctx, test.cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mux := http.NewServeMux()
+ server.Install(mux.Handle, nil, nil)
+
w := httptest.NewRecorder()
mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil))
if w.Code != http.StatusOK {
@@ -203,36 +235,3 @@
t.Errorf("got %v, want %v", got, want)
}
}
-
-func TestListModsForPaths(t *testing.T) {
- listModules = func(string) ([]listedMod, error) {
- return []listedMod{
- {
- internal.Modver{Path: "m1", Version: "v1.2.3"},
- "/dir/cache/download/m1/@v/v1.2.3.mod",
- false,
- },
- {
- internal.Modver{Path: "m2", Version: "v1.0.0"},
- "/repos/m2/go.mod",
- false,
- },
- {
- internal.Modver{Path: "indir", Version: "v2.3.4"},
- "",
- true,
- },
- }, nil
- }
- defer func() { listModules = _listModules }()
-
- gotPaths, gotCacheMods, err := listModsForPaths([]string{"m1"}, "/dir")
- if err != nil {
- t.Fatal(err)
- }
- wantPaths := []string{"/repos/m2"}
- wantCacheMods := []internal.Modver{{Path: "m1", Version: "v1.2.3"}}
- if !cmp.Equal(gotPaths, wantPaths) || !cmp.Equal(gotCacheMods, wantCacheMods) {
- t.Errorf("got\n%v, %v\nwant\n%v, %v", gotPaths, gotCacheMods, wantPaths, wantCacheMods)
- }
-}
diff --git a/internal/datasource.go b/internal/datasource.go
index 657e1ea..72b924a 100644
--- a/internal/datasource.go
+++ b/internal/datasource.go
@@ -4,7 +4,73 @@
package internal
-import "context"
+import (
+ "context"
+ "time"
+)
+
+// SearchOptions provide information used by db.Search.
+type SearchOptions struct {
+ // Maximum number of results to return (page size).
+ MaxResults int
+
+ // Offset for DB query.
+ Offset int
+
+ // Maximum number to use for total result count.
+ MaxResultCount int
+
+ // If true, perform a symbol search.
+ SearchSymbols bool
+
+ // SymbolFilter is the word in a search query with a # prefix.
+ SymbolFilter string
+}
+
+// SearchResult represents a single search result from SearchDocuments.
+type SearchResult struct {
+ Name string
+ PackagePath string
+ ModulePath string
+ Version string
+ Synopsis string
+ Licenses []string
+
+ CommitTime time.Time
+
+ // Score is used to sort items in an array of SearchResult.
+ Score float64
+
+ // NumImportedBy is the number of packages that import PackagePath.
+ NumImportedBy uint64
+
+ // SameModule is a list of SearchResults from the same module as this one,
+ // with lower scores.
+ SameModule []*SearchResult
+
+ // OtherMajor is a map from module paths with the same series path but at
+ // different major versions of this module, to major version.
+ // The major version for a non-vN module path (either 0 or 1) is computed
+ // based on the version in search documents.
+ OtherMajor map[string]int
+
+ // NumResults is the total number of packages that were returned for this
+ // search.
+ NumResults uint64
+
+ // Symbol information returned by a search request.
+ // Only populated for symbol search mode.
+ SymbolName string
+ SymbolKind SymbolKind
+ SymbolSynopsis string
+ SymbolGOOS string
+ SymbolGOARCH string
+
+ // Offset is the 0-based number of this row in the DB query results, which
+ // is the value to use in a SQL OFFSET clause to have this row be the first
+ // one returned.
+ Offset int
+}
// DataSource is the interface used by the frontend to interact with module data.
type DataSource interface {
@@ -22,10 +88,14 @@
GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *UnitMeta, err error)
// GetModuleReadme gets the readme for the module.
GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*Readme, error)
-
// GetLatestInfo gets information about the latest versions of a unit and module.
// See LatestInfo for documentation.
GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *UnitMeta) (LatestInfo, error)
+
+ // SupportsSearch reports whether this data source supports search.
+ SupportsSearch() bool
+ // Search searches for packages matching the given query.
+ Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error)
}
// LatestInfo holds information about the latest versions and paths.
diff --git a/internal/fetch/getters.go b/internal/fetch/getters.go
index 9258924..33c4eaf 100644
--- a/internal/fetch/getters.go
+++ b/internal/fetch/getters.go
@@ -13,6 +13,9 @@
"encoding/json"
"errors"
"fmt"
+ "go/doc"
+ "go/parser"
+ "go/token"
"io"
"io/fs"
"os"
@@ -26,9 +29,12 @@
"golang.org/x/mod/module"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/fuzzy"
+ "golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/source"
"golang.org/x/pkgsite/internal/version"
+ "golang.org/x/tools/go/packages"
)
// ModuleGetter gets module data.
@@ -60,6 +66,21 @@
String() string
}
+// SearchableModuleGetter is an additional interface that may be implemented by
+// ModuleGetters to support search.
+type SearchableModuleGetter interface {
+ // Search searches for packages matching the given query, returning at most
+ // limit results.
+ Search(ctx context.Context, q string, limit int) ([]*internal.SearchResult, error)
+}
+
+// VolatileModuleGetter is an additional interface that may be implemented by
+// ModuleGetters to support invalidating content.
+type VolatileModuleGetter interface {
+ // HasChanged reports whether the referenced module has changed.
+ HasChanged(context.Context, internal.ModuleInfo) (bool, error)
+}
+
type proxyModuleGetter struct {
prox *proxy.Client
src *source.Client
@@ -120,7 +141,6 @@
// NewDirectoryModuleGetter returns a ModuleGetter for reading a module from a directory.
func NewDirectoryModuleGetter(modulePath, dir string) (*directoryModuleGetter, error) {
-
if modulePath == "" {
goModBytes, err := os.ReadFile(filepath.Join(dir, "go.mod"))
if err != nil {
@@ -203,39 +223,313 @@
return fmt.Sprintf("Dir(%s, %s)", g.modulePath, g.dir)
}
-// An fsProxyModuleGetter gets modules from a directory in the filesystem that
-// is organized like the module cache, with a cache/download directory that has
-// paths that correspond to proxy URLs. An example of such a directory is
-// $(go env GOMODCACHE).
-type fsProxyModuleGetter struct {
- dir string
- allowed map[internal.Modver]bool
+// A goPackagesModuleGetter is a ModuleGetter whose source is go/packages.Load
+// from a directory in the local file system.
+type goPackagesModuleGetter struct {
+ dir string // directory from which go/packages was run
+ packages []*packages.Package // all packages
+ modules []*packages.Module // modules references by packagages; sorted by path
}
-// NewFSProxyModuleGetter returns a ModuleGetter that reads modules from a filesystem
+// NewGoPackagesModuleGetter returns a ModuleGetter that loads packages using
+// go/packages.Load(pattern), from the requested directory.
+func NewGoPackagesModuleGetter(ctx context.Context, dir string, pattern string) (*goPackagesModuleGetter, error) {
+ abs, err := filepath.Abs(dir)
+ if err != nil {
+ return nil, err
+ }
+ start := time.Now()
+ cfg := &packages.Config{
+ Context: ctx,
+ Dir: abs,
+ Mode: packages.NeedName |
+ packages.NeedModule |
+ packages.NeedCompiledGoFiles |
+ packages.NeedFiles,
+ }
+ pkgs, err := packages.Load(cfg, pattern)
+ log.Infof(ctx, "go/packages.Load(%q) loaded %d packages from %s in %v", pattern, len(pkgs), dir, time.Since(start))
+ if err != nil {
+ return nil, err
+ }
+
+ // Collect reachable modules. Modules must be sorted for search.
+ moduleSet := make(map[string]*packages.Module)
+ for _, pkg := range pkgs {
+ if pkg.Module != nil {
+ moduleSet[pkg.Module.Path] = pkg.Module
+ }
+ }
+ var modules []*packages.Module
+ for _, m := range moduleSet {
+ modules = append(modules, m)
+ }
+ sort.Slice(modules, func(i, j int) bool {
+ return modules[i].Path < modules[j].Path
+ })
+
+ return &goPackagesModuleGetter{
+ dir: abs,
+ packages: pkgs,
+ modules: modules,
+ }, nil
+}
+
+// findModule searches known modules for a module matching the provided path.
+func (g *goPackagesModuleGetter) findModule(path string) (*packages.Module, error) {
+ i := sort.Search(len(g.modules), func(i int) bool {
+ return g.modules[i].Path >= path
+ })
+ if i >= len(g.modules) || g.modules[i].Path != path {
+ return nil, fmt.Errorf("%w: no module with path %q", derrors.NotFound, path)
+ }
+ return g.modules[i], nil
+}
+
+// Info returns basic information about the module.
+//
+// For invalidation of locally edited modules, the time of the resulting
+// version is set to the latest mtime of a file referenced by any compiled file
+// in the module.
+func (g *goPackagesModuleGetter) Info(ctx context.Context, modulePath, version string) (*proxy.VersionInfo, error) {
+ m, err := g.findModule(modulePath)
+ if err != nil {
+ return nil, err
+ }
+ v := LocalVersion
+ if m.Version != "" {
+ v = m.Version
+ }
+ // Note: if we ever support loading dependencies out of the module cache, we
+ // may have a valid m.Time to use here.
+ var t time.Time
+ mtime, err := g.mtime(ctx, m)
+ if err != nil {
+ return nil, err
+ }
+ if mtime != nil {
+ t = *mtime
+ } else {
+ t = LocalCommitTime
+ }
+ return &proxy.VersionInfo{
+ Version: v,
+ Time: t,
+ }, nil
+}
+
+// mtime returns the latest modification time of a compiled Go file contained
+// in a package in the module.
+//
+// TODO(rfindley): we should probably walk the entire module directory, so that
+// we pick up new or deleted go files, but must be careful about nested
+// modules.
+func (g *goPackagesModuleGetter) mtime(ctx context.Context, m *packages.Module) (*time.Time, error) {
+ var mtime *time.Time
+ for _, pkg := range g.packages {
+ if pkg.Module != nil && pkg.Module.Path == m.Path {
+ for _, f := range pkg.CompiledGoFiles {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ fi, err := os.Stat(f)
+ if os.IsNotExist(err) {
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+ if mtime == nil || fi.ModTime().After(*mtime) {
+ modTime := fi.ModTime()
+ mtime = &modTime
+ }
+ }
+ }
+ }
+
+ // If mtime is recent, it may be unrelable as due to system time resolution
+ // we may yet receive another edit within the same tick.
+ if mtime != nil && time.Since(*mtime) < 2*time.Second {
+ return nil, nil
+ }
+
+ return mtime, nil
+}
+
+// Mod returns the contents of the module's go.mod file.
+// If the file does not exist, it returns a synthesized one.
+func (g *goPackagesModuleGetter) Mod(ctx context.Context, modulePath, version string) ([]byte, error) {
+ m, err := g.findModule(modulePath)
+ if err != nil {
+ return nil, err
+ }
+ if m.Dir == "" {
+ return nil, fmt.Errorf("module %q missing dir", modulePath)
+ }
+ data, err := os.ReadFile(filepath.Join(m.Dir, "go.mod"))
+ if errors.Is(err, os.ErrNotExist) {
+ return []byte(fmt.Sprintf("module %s\n", modulePath)), nil
+ }
+ return data, err
+}
+
+// ContentDir returns an fs.FS for the module's contents.
+func (g *goPackagesModuleGetter) ContentDir(ctx context.Context, modulePath, version string) (fs.FS, error) {
+ m, err := g.findModule(modulePath)
+ if err != nil {
+ return nil, err
+ }
+ if m.Dir == "" {
+ return nil, fmt.Errorf("module %q missing dir", modulePath)
+ }
+ return os.DirFS(m.Dir), nil
+}
+
+// SourceInfo returns a source.Info that will link to the files in the
+// directory. The files will be under /files/directory/modulePath, with no
+// version.
+func (g *goPackagesModuleGetter) SourceInfo(ctx context.Context, modulePath, _ string) (*source.Info, error) {
+ m, err := g.findModule(modulePath)
+ if err != nil {
+ return nil, err
+ }
+ if m.Dir == "" {
+ return nil, fmt.Errorf("module %q missing dir", modulePath)
+ }
+ p := path.Join(filepath.ToSlash(g.dir), modulePath)
+ return source.FilesInfo(p), nil
+}
+
+// Open implements the fs.FS interface, matching the path name to a loaded
+// module.
+func (g *goPackagesModuleGetter) Open(name string) (fs.File, error) {
+ var bestMatch *packages.Module
+ for _, m := range g.modules {
+ if strings.HasPrefix(name+"/", m.Path+"/") {
+ if bestMatch == nil || m.Path > bestMatch.Path {
+ bestMatch = m
+ }
+ }
+ }
+ if bestMatch == nil {
+ return nil, fmt.Errorf("no module matching %s", name)
+ }
+ suffix := strings.TrimPrefix(name, bestMatch.Path)
+ suffix = strings.TrimPrefix(suffix, "/")
+ filename := filepath.Join(bestMatch.Dir, filepath.FromSlash(suffix))
+ return os.Open(filename)
+}
+
+func (g *goPackagesModuleGetter) SourceFS() (string, fs.FS) {
+ return filepath.ToSlash(g.dir), g
+}
+
+// For testing.
+func (g *goPackagesModuleGetter) String() string {
+ return fmt.Sprintf("Dir(%s)", g.dir)
+}
+
+// Search implements a crude search, using fuzzy matching to match loaded
+// packages.
+//
+// It parses file headers to produce a synopsis of results.
+func (g *goPackagesModuleGetter) Search(ctx context.Context, query string, limit int) ([]*internal.SearchResult, error) {
+ matcher := fuzzy.NewSymbolMatcher(query)
+
+ type scoredPackage struct {
+ pkg *packages.Package
+ score float64
+ }
+
+ var pkgs []scoredPackage
+ for _, pkg := range g.packages {
+ i, score := matcher.Match([]string{pkg.PkgPath})
+ if i < 0 {
+ continue
+ }
+ pkgs = append(pkgs, scoredPackage{pkg, score})
+ }
+
+ // Sort and truncate results before parsing, to save on work.
+ sort.Slice(pkgs, func(i, j int) bool {
+ return pkgs[i].score > pkgs[j].score
+ })
+
+ if len(pkgs) > limit {
+ pkgs = pkgs[:limit]
+ }
+
+ var results []*internal.SearchResult
+ for i, pkg := range pkgs {
+ result := &internal.SearchResult{
+ Name: pkg.pkg.Name,
+ PackagePath: pkg.pkg.PkgPath,
+ Score: pkg.score,
+ Offset: i,
+ }
+ if pkg.pkg.Module != nil {
+ result.ModulePath = pkg.pkg.Module.Path
+ result.Version = pkg.pkg.Module.Version
+ }
+ for _, file := range pkg.pkg.CompiledGoFiles {
+ mode := parser.PackageClauseOnly | parser.ParseComments
+ f, err := parser.ParseFile(token.NewFileSet(), file, nil, mode)
+ if err != nil {
+ continue
+ }
+ if f.Doc != nil {
+ result.Synopsis = doc.Synopsis(f.Doc.Text())
+ }
+ }
+ results = append(results, result)
+ }
+ return results, nil
+}
+
+// HasChanged stats the filesystem to see if content has changed for the
+// provided module. It compares the latest mtime of package files to the time
+// recorded in info.CommitTime, which stores the last observed mtime.
+func (g *goPackagesModuleGetter) HasChanged(ctx context.Context, info internal.ModuleInfo) (bool, error) {
+ m, err := g.findModule(info.ModulePath)
+ if err != nil {
+ return false, err
+ }
+ mtime, err := g.mtime(ctx, m)
+ if err != nil {
+ return false, err
+ }
+ return mtime == nil || mtime.After(info.CommitTime), nil
+}
+
+// A modCacheModuleGetter gets modules from a directory in the filesystem that
+// is organized like the module cache, with a cache/download directory that has
+// paths that correspond to proxy URLs. An example of such a directory is $(go
+// env GOMODCACHE).
+//
+// TODO(rfindley): it would be easy and useful to add support for Search to
+// this getter.
+type modCacheModuleGetter struct {
+ dir string
+}
+
+// NewModCacheGetter returns a ModuleGetter that reads modules from a filesystem
// directory organized like the proxy.
// If allowed is non-empty, only module@versions in allowed are served; others
// result in NotFound errors.
-func NewFSProxyModuleGetter(dir string, allowed []internal.Modver) (_ *fsProxyModuleGetter, err error) {
+func NewModCacheGetter(dir string) (_ *modCacheModuleGetter, err error) {
defer derrors.Wrap(&err, "NewFSProxyModuleGetter(%q)", dir)
abs, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
- g := &fsProxyModuleGetter{dir: abs}
- if len(allowed) > 0 {
- g.allowed = map[internal.Modver]bool{}
- for _, a := range allowed {
- g.allowed[a] = true
- }
- }
+ g := &modCacheModuleGetter{dir: abs}
return g, nil
}
// Info returns basic information about the module.
-func (g *fsProxyModuleGetter) Info(ctx context.Context, path, vers string) (_ *proxy.VersionInfo, err error) {
- defer derrors.Wrap(&err, "fsProxyModuleGetter.Info(%q, %q)", path, vers)
+func (g *modCacheModuleGetter) Info(ctx context.Context, path, vers string) (_ *proxy.VersionInfo, err error) {
+ defer derrors.Wrap(&err, "modCacheGetter.Info(%q, %q)", path, vers)
if vers == version.Latest {
vers, err = g.latestVersion(path)
@@ -262,8 +556,8 @@
}
// Mod returns the contents of the module's go.mod file.
-func (g *fsProxyModuleGetter) Mod(ctx context.Context, path, vers string) (_ []byte, err error) {
- defer derrors.Wrap(&err, "fsProxyModuleGetter.Mod(%q, %q)", path, vers)
+func (g *modCacheModuleGetter) Mod(ctx context.Context, path, vers string) (_ []byte, err error) {
+ defer derrors.Wrap(&err, "modCacheModuleGetter.Mod(%q, %q)", path, vers)
if vers == version.Latest {
vers, err = g.latestVersion(path)
@@ -282,8 +576,8 @@
}
// ContentDir returns an fs.FS for the module's contents.
-func (g *fsProxyModuleGetter) ContentDir(ctx context.Context, path, vers string) (_ fs.FS, err error) {
- defer derrors.Wrap(&err, "fsProxyModuleGetter.ContentDir(%q, %q)", path, vers)
+func (g *modCacheModuleGetter) ContentDir(ctx context.Context, path, vers string) (_ fs.FS, err error) {
+ defer derrors.Wrap(&err, "modCacheModuleGetter.ContentDir(%q, %q)", path, vers)
if vers == version.Latest {
vers, err = g.latestVersion(path)
@@ -305,19 +599,19 @@
// SourceInfo returns a source.Info that will create /files links to modules in
// the cache.
-func (g *fsProxyModuleGetter) SourceInfo(ctx context.Context, mpath, version string) (*source.Info, error) {
+func (g *modCacheModuleGetter) SourceInfo(ctx context.Context, mpath, version string) (*source.Info, error) {
return source.FilesInfo(path.Join(g.dir, mpath+"@"+version)), nil
}
// SourceFS returns the absolute path to the cache, and an FS that retrieves
// files from it.
-func (g *fsProxyModuleGetter) SourceFS() (string, fs.FS) {
+func (g *modCacheModuleGetter) SourceFS() (string, fs.FS) {
return filepath.ToSlash(g.dir), os.DirFS(g.dir)
}
// latestVersion gets the latest version that is in the directory.
-func (g *fsProxyModuleGetter) latestVersion(modulePath string) (_ string, err error) {
- defer derrors.Wrap(&err, "fsProxyModuleGetter.latestVersion(%q)", modulePath)
+func (g *modCacheModuleGetter) latestVersion(modulePath string) (_ string, err error) {
+ defer derrors.Wrap(&err, "modCacheModuleGetter.latestVersion(%q)", modulePath)
dir, err := g.moduleDir(modulePath)
if err != nil {
@@ -333,27 +627,13 @@
var versions []string
for _, z := range zips {
vers := strings.TrimSuffix(filepath.Base(z), ".zip")
- if g.allow(modulePath, vers) {
- versions = append(versions, vers)
- }
- }
- if len(versions) == 0 {
- return "", fmt.Errorf("no allowed versions form module %q: %w", modulePath, derrors.NotFound)
+ versions = append(versions, vers)
}
return version.LatestOf(versions), nil
}
-// allow reports whether the module path and version may be served.
-func (g *fsProxyModuleGetter) allow(path, vers string) bool {
- return g.allowed == nil || g.allowed[internal.Modver{Path: path, Version: vers}]
-}
-
-func (g *fsProxyModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
- defer derrors.Wrap(&err, "fsProxyModuleGetter.readFile(%q, %q, %q)", path, version, suffix)
-
- if !g.allow(path, version) {
- return nil, fmt.Errorf("%s@%s not allowed: %w", path, version, derrors.NotFound)
- }
+func (g *modCacheModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
+ defer derrors.Wrap(&err, "modCacheModuleGetter.readFile(%q, %q, %q)", path, version, suffix)
f, err := g.openFile(path, version, suffix)
if err != nil {
@@ -363,7 +643,7 @@
return io.ReadAll(f)
}
-func (g *fsProxyModuleGetter) openFile(path, version, suffix string) (_ *os.File, err error) {
+func (g *modCacheModuleGetter) openFile(path, version, suffix string) (_ *os.File, err error) {
epath, err := g.escapedPath(path, version, suffix)
if err != nil {
return nil, err
@@ -378,7 +658,7 @@
return f, nil
}
-func (g *fsProxyModuleGetter) escapedPath(modulePath, version, suffix string) (string, error) {
+func (g *modCacheModuleGetter) escapedPath(modulePath, version, suffix string) (string, error) {
dir, err := g.moduleDir(modulePath)
if err != nil {
return "", err
@@ -390,7 +670,7 @@
return filepath.Join(dir, fmt.Sprintf("%s.%s", ev, suffix)), nil
}
-func (g *fsProxyModuleGetter) moduleDir(modulePath string) (string, error) {
+func (g *modCacheModuleGetter) moduleDir(modulePath string) (string, error) {
ep, err := module.EscapePath(modulePath)
if err != nil {
return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
@@ -399,14 +679,6 @@
}
// For testing.
-func (g *fsProxyModuleGetter) String() string {
- if g.allowed == nil {
- return fmt.Sprintf("FSProxy(%s)", g.dir)
- }
- var as []string
- for a := range g.allowed {
- as = append(as, a.String())
- }
- sort.Strings(as)
- return fmt.Sprintf("FSProxy(%s, %s)", g.dir, strings.Join(as, ","))
+func (g *modCacheModuleGetter) String() string {
+ return fmt.Sprintf("FSProxy(%s)", g.dir)
}
diff --git a/internal/fetch/getters_test.go b/internal/fetch/getters_test.go
index c29bd9a..77fe56b 100644
--- a/internal/fetch/getters_test.go
+++ b/internal/fetch/getters_test.go
@@ -8,14 +8,16 @@
"context"
"errors"
"io"
+ "io/fs"
+ "os"
"path/filepath"
"testing"
"time"
"github.com/google/go-cmp/cmp"
- "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/proxy"
+ "golang.org/x/pkgsite/internal/testing/testhelper"
"golang.org/x/pkgsite/internal/version"
)
@@ -34,6 +36,186 @@
}
}
+const multiModule = `
+-- go.work --
+go 1.21
+
+use (
+ ./foo
+ ./bar
+)
+
+-- foo/go.mod --
+module foo.com/foo
+
+go 1.21
+-- foo/foolog/f.go --
+package foolog
+
+const Log = 1
+-- bar/go.mod --
+module bar.com/bar
+
+go 1.20
+-- bar/barlog/b.go --
+package barlog
+
+const Log = 1
+`
+
+func TestGoPackagesModuleGetter(t *testing.T) {
+ modulePaths := map[string]string{ // dir -> module path
+ "foo": "foo.com/foo",
+ "bar": "bar.com/bar",
+ }
+
+ tests := []struct {
+ name string
+ dir string
+ }{
+ {"work dir", "."},
+ {"module dir", "foo"},
+ {"nested package dir", "foo/foolog"},
+ }
+
+ ctx := context.Background()
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ tempDir, files := testhelper.WriteTxtarToTempDir(t, multiModule)
+ dir := filepath.Join(tempDir, test.dir)
+
+ g, err := NewGoPackagesModuleGetter(ctx, dir, "all")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for moduleDir, modulePath := range modulePaths {
+ t.Run("info", func(t *testing.T) {
+ got, err := g.Info(ctx, modulePath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, want := got.Version, LocalVersion; got != want {
+ t.Errorf("Info(%s): got version %s, want %s", modulePath, got, want)
+ }
+ })
+
+ mod := files[moduleDir+"/go.mod"]
+ t.Run("mod", func(t *testing.T) {
+ got, err := g.Mod(ctx, modulePath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(mod, string(got)); diff != "" {
+ t.Errorf("Mod(%q) mismatch [-want +got]:\n%s", modulePath, diff)
+ }
+ })
+
+ t.Run("contentdir", func(t *testing.T) {
+ fsys, err := g.ContentDir(ctx, modulePath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Just check that the go.mod file is there and has the right contents.
+ got, err := fs.ReadFile(fsys, "go.mod")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(mod, string(got)); diff != "" {
+ t.Errorf("fs.ReadFile(ContentDir(%q), %q) mismatch [-want +got]:\n%s", modulePath, "go.mod", diff)
+ }
+ })
+
+ t.Run("search", func(t *testing.T) {
+ tests := []struct {
+ query string
+ want []string
+ }{
+ {"log", []string{"barlog", "foolog"}},
+ {"barlog", []string{"barlog"}},
+ {"xxxxxx", nil},
+ }
+
+ for _, test := range tests {
+ results, err := g.Search(ctx, test.query, 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got []string
+ for _, r := range results {
+ got = append(got, r.Name)
+ }
+ if diff := cmp.Diff(test.want, got); diff != "" {
+ t.Errorf("Search(%s) mismatch [-want +got]:\n%s", test.query, diff)
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGoPackagesModuleGetter_Invalidation(t *testing.T) {
+ ctx := context.Background()
+
+ tempDir, _ := testhelper.WriteTxtarToTempDir(t, multiModule)
+
+ // Sleep before fetching the initial info, so that the written mtime will be
+ // considered reliable enough for caching by the getter.
+ time.Sleep(3 * time.Second)
+
+ g, err := NewGoPackagesModuleGetter(ctx, tempDir, "all")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ const fooPath = "foo.com/foo"
+ foo1, err := g.Info(ctx, fooPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ foo2, err := g.Info(ctx, fooPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmp.Equal(foo1, foo2) {
+ t.Errorf("Info(%q) returned inconsistent results: %v != %v", fooPath, foo1, foo2)
+ }
+
+ const barPath = "bar.com/bar"
+ bar1, err := g.Info(ctx, barPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ bar2, err := g.Info(ctx, barPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmp.Equal(bar1, bar2) {
+ t.Errorf("Info(%q) returned inconsistent results: %v != %v", barPath, bar1, bar2)
+ }
+
+ fpath := filepath.Join(tempDir, "foo", "foolog", "f.go")
+ newContent := []byte("package foolog; const Log = 3")
+ if err := os.WriteFile(fpath, newContent, 0600); err != nil {
+ t.Fatal(err)
+ }
+ foo3, err := g.Info(ctx, fooPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if cmp.Equal(foo1, foo3) {
+ t.Errorf("Info(%q) results unexpectedly match: %v == %v", fooPath, foo1, foo3)
+ }
+ bar3, err := g.Info(ctx, barPath, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmp.Equal(bar1, bar2) {
+ t.Errorf("Info(%q) returned inconsistent results: %v != %v", barPath, bar1, bar3)
+ }
+}
+
func TestEscapedPath(t *testing.T) {
for _, test := range []struct {
path, version, suffix string
@@ -48,7 +230,7 @@
"dir/cache/download/github.com/a!bc/@v/v2.3.4.zip",
},
} {
- g, err := NewFSProxyModuleGetter("dir", nil)
+ g, err := NewModCacheGetter("dir")
if err != nil {
t.Fatal(err)
}
@@ -77,7 +259,7 @@
if err != nil {
t.Fatal(err)
}
- g, err := NewFSProxyModuleGetter("testdata/modcache", nil)
+ g, err := NewModCacheGetter("testdata/modcache")
if err != nil {
t.Fatal(err)
}
@@ -143,37 +325,3 @@
}
})
}
-
-func TestFSProxyGetterAllowed(t *testing.T) {
- ctx := context.Background()
- // The cache contains:
- // github.com/jackc/pgio@v1.0.0
- // modcache.com@v1.0.0
- // Allow only the latter.
- allow := []internal.Modver{{Path: "modcache.com", Version: "v1.0.0"}}
- g, err := NewFSProxyModuleGetter("testdata/modcache", allow)
- if err != nil {
- t.Fatal(err)
- }
-
- check := func(path, vers string, wantOK bool) {
- t.Helper()
- _, err := g.Info(ctx, path, vers)
- if (err == nil) != wantOK {
- s := "a"
- if wantOK {
- s = "no"
- }
- t.Errorf("got %v; wanted %s error", err, s)
- } else if err != nil && !errors.Is(err, derrors.NotFound) {
- t.Errorf("got %v, wanted NotFound", err)
- }
- }
-
- // We can get modcache.com.
- check("modcache.com", "v1.0.0", true)
- check("modcache.com", "latest", true)
- // We get NotFound for jackc.
- check("github.com/jackc/pgio", "v1.0.0", false)
- check("github.com/jackc/pgio", "latest", false)
-}
diff --git a/internal/fetchdatasource/fetchdatasource.go b/internal/fetchdatasource/fetchdatasource.go
index 87a69fb..efcd20e 100644
--- a/internal/fetchdatasource/fetchdatasource.go
+++ b/internal/fetchdatasource/fetchdatasource.go
@@ -11,6 +11,7 @@
"context"
"errors"
"fmt"
+ "sort"
"strconv"
"strings"
"time"
@@ -22,6 +23,7 @@
"golang.org/x/pkgsite/internal/fetch"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/proxy"
+ "golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/version"
)
@@ -61,6 +63,7 @@
// cacheEntry holds a fetched module or an error, if the fetch failed.
type cacheEntry struct {
+ g fetch.ModuleGetter
module *internal.Module
err error
}
@@ -68,21 +71,21 @@
const maxCachedModules = 100
// cacheGet returns information from the cache if it is present, and (nil, nil) otherwise.
-func (ds *FetchDataSource) cacheGet(path, version string) (*internal.Module, error) {
+func (ds *FetchDataSource) cacheGet(path, version string) (fetch.ModuleGetter, *internal.Module, error) {
// Look for an exact match first, then use LocalVersion, as for a
// directory-based or GOPATH-mode module.
for _, v := range []string{version, fetch.LocalVersion} {
if e, ok := ds.cache.Get(internal.Modver{Path: path, Version: v}); ok {
e := e.(cacheEntry)
- return e.module, e.err
+ return e.g, e.module, e.err
}
}
- return nil, nil
+ return nil, nil, nil
}
// cachePut puts information into the cache.
-func (ds *FetchDataSource) cachePut(path, version string, m *internal.Module, err error) {
- ds.cache.Add(internal.Modver{Path: path, Version: version}, cacheEntry{m, err})
+func (ds *FetchDataSource) cachePut(g fetch.ModuleGetter, path, version string, m *internal.Module, err error) {
+ ds.cache.Add(internal.Modver{Path: path, Version: version}, cacheEntry{g, m, err})
}
// getModule gets the module at the given path and version. It first checks the
@@ -90,15 +93,30 @@
func (ds *FetchDataSource) getModule(ctx context.Context, modulePath, vers string) (_ *internal.Module, err error) {
defer derrors.Wrap(&err, "FetchDataSource.getModule(%q, %q)", modulePath, vers)
- mod, err := ds.cacheGet(modulePath, vers)
- if mod != nil || err != nil {
- return mod, err
+ g, mod, err := ds.cacheGet(modulePath, vers)
+ if err != nil {
+ return nil, err
+ }
+ if mod != nil {
+ // For getters supporting invalidation, check whether cached contents have
+ // changed.
+ v, ok := g.(fetch.VolatileModuleGetter)
+ if !ok {
+ return mod, nil
+ }
+ hasChanged, err := v.HasChanged(ctx, mod.ModuleInfo)
+ if err != nil {
+ return nil, err
+ }
+ if !hasChanged {
+ return mod, nil
+ }
}
// There can be a benign race here, where two goroutines both fetch the same
// module. At worst some work will be duplicated, but if that turns out to
// be a problem we could use golang.org/x/sync/singleflight.
- m, err := ds.fetch(ctx, modulePath, vers)
+ m, g, err := ds.fetch(ctx, modulePath, vers)
if m != nil && ds.opts.ProxyClientForLatest != nil {
// Use the go.mod file at the raw latest version to fill in deprecation
// and retraction information. Ignore any problems getting the
@@ -118,11 +136,11 @@
// Cache both successes and failures, but not cancellations.
if !errors.Is(err, context.Canceled) {
- ds.cachePut(modulePath, vers, m, err)
+ ds.cachePut(g, modulePath, vers, m, err)
// Cache the resolved version of "latest" too. A useful optimization
// because the frontend redirects "latest", resulting in another fetch.
if m != nil && vers == version.Latest {
- ds.cachePut(modulePath, m.Version, m, err)
+ ds.cachePut(g, modulePath, m.Version, m, err)
}
}
return m, err
@@ -130,11 +148,11 @@
// fetch fetches a module using the configured ModuleGetters.
// It tries each getter in turn until it finds one that has the module.
-func (ds *FetchDataSource) fetch(ctx context.Context, modulePath, version string) (_ *internal.Module, err error) {
+func (ds *FetchDataSource) fetch(ctx context.Context, modulePath, version string) (_ *internal.Module, g fetch.ModuleGetter, err error) {
log.Infof(ctx, "FetchDataSource: fetching %s@%s", modulePath, version)
start := time.Now()
defer func() {
- log.Infof(ctx, "FetchDataSource: fetched %s@%s in %s with error %v", modulePath, version, time.Since(start), err)
+ log.Infof(ctx, "FetchDataSource: fetched %s@%s using %T in %s with error %v", modulePath, version, g, time.Since(start), err)
}()
for _, g := range ds.opts.Getters {
fr := fetch.FetchModule(ctx, modulePath, version, g)
@@ -148,13 +166,22 @@
} else {
m.RemoveNonRedistributableData()
}
- return m, nil
+ // There is special handling in FetchModule for the standard library,
+ // that bypasses the getter g. Don't record g as having fetch std.
+ //
+ // TODO(rfindley): it would be cleaner if the standard library could be
+ // its own module getter. This could also allow the go/packages getter to
+ // serve existing on-disk content for std. See also golang/go#58923.
+ if modulePath == stdlib.ModulePath {
+ g = nil
+ }
+ return m, g, nil
}
if !errors.Is(fr.Error, derrors.NotFound) {
- return nil, fr.Error
+ return nil, g, fr.Error
}
}
- return nil, fmt.Errorf("%s@%s: %w", modulePath, version, derrors.NotFound)
+ return nil, nil, fmt.Errorf("%s@%s: %w", modulePath, version, derrors.NotFound)
}
func (ds *FetchDataSource) populateUnitSubdirectories(u *internal.Unit, m *internal.Module) {
@@ -350,3 +377,47 @@
func (*FetchDataSource) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*internal.Readme, error) {
return nil, nil
}
+
+// SupportsSearch reports whether any of the configured Getters are searchable.
+func (ds *FetchDataSource) SupportsSearch() bool {
+ for _, g := range ds.opts.Getters {
+ if _, ok := g.(fetch.SearchableModuleGetter); ok {
+ return true
+ }
+ }
+ return false
+}
+
+// Search delegates search to any configured getters that support the
+// SearchableModuleGetter interface, merging their results.
+func (ds *FetchDataSource) Search(ctx context.Context, q string, opts internal.SearchOptions) (_ []*internal.SearchResult, err error) {
+ var results []*internal.SearchResult
+ // Since results are potentially merged from multiple sources, we can't know
+ // a priori how many results will be used from any particular getter.
+ //
+ // Offset+MaxResults is an upper bound.
+ limit := opts.Offset + opts.MaxResults
+ for _, g := range ds.opts.Getters {
+ if s, ok := g.(fetch.SearchableModuleGetter); ok {
+ rs, err := s.Search(ctx, q, limit)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, rs...)
+ }
+ }
+ sort.Slice(results, func(i, j int) bool {
+ return results[i].Score > results[j].Score
+ })
+ if opts.Offset > 0 {
+ if len(results) < opts.Offset {
+ return nil, nil
+ }
+ results = results[opts.Offset:]
+ }
+ if opts.MaxResults > 0 && len(results) > opts.MaxResults {
+ results = results[:opts.MaxResults]
+ }
+
+ return results, nil
+}
diff --git a/internal/fetchdatasource/fetchdatasource_test.go b/internal/fetchdatasource/fetchdatasource_test.go
index e433428..ecc63fd 100644
--- a/internal/fetchdatasource/fetchdatasource_test.go
+++ b/internal/fetchdatasource/fetchdatasource_test.go
@@ -95,13 +95,14 @@
dirs []string
getters []fetch.ModuleGetter
)
+ ctx := context.Background()
for _, module := range modules {
directory, err := testhelper.CreateTestDirectory(module)
if err != nil {
log.Fatal(err)
}
dirs = append(dirs, directory)
- mg, err := fetch.NewDirectoryModuleGetter("", directory)
+ mg, err := fetch.NewGoPackagesModuleGetter(ctx, directory, "./...")
if err != nil {
log.Fatal(err)
}
@@ -355,7 +356,6 @@
ModuleInfo: internal.ModuleInfo{
ModulePath: "github.com/my/module",
Version: fetch.LocalVersion,
- CommitTime: fetch.LocalCommitTime,
IsRedistributable: true,
HasGoMod: true,
SourceInfo: sourceInfo,
@@ -372,7 +372,6 @@
ModuleInfo: internal.ModuleInfo{
ModulePath: "github.com/my/module",
Version: fetch.LocalVersion,
- CommitTime: fetch.LocalCommitTime,
IsRedistributable: true,
HasGoMod: true,
SourceInfo: sourceInfo,
@@ -390,7 +389,6 @@
ModulePath: "github.com/my/module",
IsRedistributable: true,
Version: fetch.LocalVersion,
- CommitTime: fetch.LocalCommitTime,
HasGoMod: true,
SourceInfo: sourceInfo,
},
@@ -407,7 +405,6 @@
ModuleInfo: internal.ModuleInfo{
ModulePath: "github.com/my/module",
Version: fetch.LocalVersion,
- CommitTime: fetch.LocalCommitTime,
IsRedistributable: true,
HasGoMod: true,
SourceInfo: sourceInfo,
@@ -453,7 +450,12 @@
if !matched {
t.Errorf("RepoURL: got %q, want match of %q", gotURL, wantRegexp)
}
- diff := cmp.Diff(test.want, got, cmp.AllowUnexported(source.Info{}), cmpopts.IgnoreFields(source.Info{}, "repoURL"))
+ opts := []cmp.Option{
+ cmp.AllowUnexported(source.Info{}),
+ cmpopts.IgnoreFields(source.Info{}, "repoURL"),
+ cmpopts.IgnoreFields(internal.ModuleInfo{}, "CommitTime"), // commit time is volatile, based on file mtimes
+ }
+ diff := cmp.Diff(test.want, got, opts...)
if diff != "" {
t.Errorf("mismatch (-want +got):\n%s", diff)
}
@@ -564,8 +566,8 @@
func TestCache(t *testing.T) {
ds := Options{}.New()
m1 := &internal.Module{}
- ds.cachePut("m1", fetch.LocalVersion, m1, nil)
- ds.cachePut("m2", "v1.0.0", nil, derrors.NotFound)
+ ds.cachePut(nil, "m1", fetch.LocalVersion, m1, nil)
+ ds.cachePut(nil, "m2", "v1.0.0", nil, derrors.NotFound)
for _, test := range []struct {
path, version string
@@ -577,7 +579,7 @@
{"m2", "v1.0.0", nil, derrors.NotFound},
{"m3", "v1.0.0", nil, nil},
} {
- gotm, gote := ds.cacheGet(test.path, test.version)
+ _, gotm, gote := ds.cacheGet(test.path, test.version)
if gotm != test.wantm || gote != test.wante {
t.Errorf("%s@%s: got (%v, %v), want (%v, %v)", test.path, test.version, gotm, gote, test.wantm, test.wante)
}
diff --git a/internal/fetchdatasource/gopath_getter.go b/internal/fetchdatasource/gopath_getter.go
deleted file mode 100644
index 8b4b9a3..0000000
--- a/internal/fetchdatasource/gopath_getter.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package fetchdatasource
-
-import (
- "fmt"
- "os"
- "path/filepath"
-
- "golang.org/x/pkgsite/internal/derrors"
- "golang.org/x/pkgsite/internal/fetch"
-)
-
-// NewGOPATHModuleGetter returns a module getter that uses the GOPATH
-// environment variable to find the module with the given import path.
-func NewGOPATHModuleGetter(importPath string) (_ fetch.ModuleGetter, err error) {
- defer derrors.Wrap(&err, "NewGOPATHModuleGetter(%q)", importPath)
-
- dir := getFullPath(importPath)
- if dir == "" {
- return nil, fmt.Errorf("path %s doesn't exist: %w", importPath, derrors.NotFound)
- }
- return fetch.NewDirectoryModuleGetter(importPath, dir)
-}
-
-// getFullPath takes an import path, tests it relative to each GOPATH, and returns
-// a full path to the module. If the given import path doesn't exist in any GOPATH,
-// an empty string is returned.
-func getFullPath(modulePath string) string {
- gopaths := filepath.SplitList(os.Getenv("GOPATH"))
- for _, gopath := range gopaths {
- path := filepath.Join(gopath, "src", modulePath)
- info, err := os.Stat(path)
- if err == nil && info.IsDir() {
- return path
- }
- }
- return ""
-}
diff --git a/internal/frontend/search.go b/internal/frontend/search.go
index 5a0778c..a05e350 100644
--- a/internal/frontend/search.go
+++ b/internal/frontend/search.go
@@ -62,8 +62,8 @@
if r.Method != http.MethodGet && r.Method != http.MethodHead {
return nil, &serverError{status: http.StatusMethodNotAllowed}
}
- db, ok := ds.(*postgres.DB)
- if !ok {
+
+ if !ds.SupportsSearch() {
// The proxydatasource does not support the imported by page.
return nil, datasourceNotSupportedErr()
}
@@ -129,7 +129,7 @@
if len(filters) > 0 {
symbol = filters[0]
}
- page, err := fetchSearchPage(ctx, db, cq, symbol, pageParams, mode == searchModeSymbol, vulnClient)
+ page, err := fetchSearchPage(ctx, ds, cq, symbol, pageParams, mode == searchModeSymbol, vulnClient)
if err != nil {
// Instead of returning a 500, return a 408, since symbol searches may
// timeout for very popular symbols.
@@ -231,13 +231,13 @@
// fetchSearchPage fetches data matching the search query from the database and
// returns a SearchPage.
-func fetchSearchPage(ctx context.Context, db *postgres.DB, cq, symbol string,
+func fetchSearchPage(ctx context.Context, ds internal.DataSource, cq, symbol string,
pageParams paginationParams, searchSymbols bool, vulnClient *vuln.Client) (*SearchPage, error) {
maxResultCount := maxSearchOffset + pageParams.limit
// Pageless search: always start from the beginning.
offset := 0
- dbresults, err := db.Search(ctx, cq, postgres.SearchOptions{
+ dbresults, err := ds.Search(ctx, cq, postgres.SearchOptions{
MaxResults: pageParams.limit,
Offset: offset,
MaxResultCount: maxResultCount,
diff --git a/internal/fuzzy/fuzzy.go b/internal/fuzzy/fuzzy.go
new file mode 100644
index 0000000..6f95b45
--- /dev/null
+++ b/internal/fuzzy/fuzzy.go
@@ -0,0 +1,239 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fuzzy
+
+// Copied from x/tools/internal/fuzzy.
+
+import (
+ "unicode"
+)
+
+// SymbolMatcher implements a fuzzy matching algorithm optimized for Go symbols
+// of the form:
+//
+// example.com/path/to/package.object.field
+//
+// Knowing that we are matching symbols like this allows us to make the
+// following optimizations:
+// - We can incorporate right-to-left relevance directly into the score
+// calculation.
+// - We can match from right to left, discarding leading bytes if the input is
+// too long.
+// - We just take the right-most match without losing too much precision. This
+// allows us to use an O(n) algorithm.
+// - We can operate directly on chunked strings; in many cases we will
+// be storing the package path and/or package name separately from the
+// symbol or identifiers, so doing this avoids allocating strings.
+// - We can return the index of the right-most match, allowing us to trim
+// irrelevant qualification.
+//
+// This implementation is experimental, serving as a reference fast algorithm
+// to compare to the fuzzy algorithm implemented by Matcher.
+type SymbolMatcher struct {
+ // Using buffers of length 256 is both a reasonable size for most qualified
+ // symbols, and makes it easy to avoid bounds checks by using uint8 indexes.
+ pattern [256]rune
+ patternLen uint8
+ inputBuffer [256]rune // avoid allocating when considering chunks
+ roles [256]uint32 // which roles does a rune play (word start, etc.)
+ segments [256]uint8 // how many segments from the right is each rune
+}
+
+const (
+ segmentStart uint32 = 1 << iota
+ wordStart
+ separator
+)
+
+// NewSymbolMatcher creates a SymbolMatcher that may be used to match the given
+// search pattern.
+//
+// Currently this matcher only accepts case-insensitive fuzzy patterns.
+//
+// An empty pattern matches no input.
+func NewSymbolMatcher(pattern string) *SymbolMatcher {
+ m := &SymbolMatcher{}
+ for _, p := range pattern {
+ m.pattern[m.patternLen] = unicode.ToLower(p)
+ m.patternLen++
+ if m.patternLen == 255 || int(m.patternLen) == len(pattern) {
+ // break at 255 so that we can represent patternLen with a uint8.
+ break
+ }
+ }
+ return m
+}
+
+// Match looks for the right-most match of the search pattern within the symbol
+// represented by concatenating the given chunks, returning its offset and
+// score.
+//
+// If a match is found, the first return value will hold the absolute byte
+// offset within all chunks for the start of the symbol. In other words, the
+// index of the match within strings.Join(chunks, ""). If no match is found,
+// the first return value will be -1.
+//
+// The second return value will be the score of the match, which is always
+// between 0 and 1, inclusive. A score of 0 indicates no match.
+func (m *SymbolMatcher) Match(chunks []string) (int, float64) {
+ // Explicit behavior for an empty pattern.
+ //
+ // As a minor optimization, this also avoids nilness checks later on, since
+ // the compiler can prove that m != nil.
+ if m.patternLen == 0 {
+ return -1, 0
+ }
+
+ // First phase: populate the input buffer with lower-cased runes.
+ //
+ // We could also check for a forward match here, but since we'd have to write
+ // the entire input anyway this has negligible impact on performance.
+
+ var (
+ inputLen = uint8(0)
+ modifiers = wordStart | segmentStart
+ )
+
+input:
+ for _, chunk := range chunks {
+ for _, r := range chunk {
+ if r == '.' || r == '/' {
+ modifiers |= separator
+ }
+ // optimization: avoid calls to unicode.ToLower, which can't be inlined.
+ l := r
+ if r <= unicode.MaxASCII {
+ if 'A' <= r && r <= 'Z' {
+ l = r + 'a' - 'A'
+ }
+ } else {
+ l = unicode.ToLower(r)
+ }
+ if l != r {
+ modifiers |= wordStart
+ }
+ m.inputBuffer[inputLen] = l
+ m.roles[inputLen] = modifiers
+ inputLen++
+ if m.roles[inputLen-1]&separator != 0 {
+ modifiers = wordStart | segmentStart
+ } else {
+ modifiers = 0
+ }
+ // TODO: we should prefer the right-most input if it overflows, rather
+ // than the left-most as we're doing here.
+ if inputLen == 255 {
+ break input
+ }
+ }
+ }
+
+ // Second phase: find the right-most match, and count segments from the
+ // right.
+
+ var (
+ pi = uint8(m.patternLen - 1) // pattern index
+ p = m.pattern[pi] // pattern rune
+ start = -1 // start offset of match
+ rseg = uint8(0)
+ )
+ const maxSeg = 3 // maximum number of segments from the right to count, for scoring purposes.
+
+ for ii := inputLen - 1; ; ii-- {
+ r := m.inputBuffer[ii]
+ if rseg < maxSeg && m.roles[ii]&separator != 0 {
+ rseg++
+ }
+ m.segments[ii] = rseg
+ if p == r {
+ if pi == 0 {
+ start = int(ii)
+ break
+ }
+ pi--
+ p = m.pattern[pi]
+ }
+ // Don't check ii >= 0 in the loop condition: ii is a uint8.
+ if ii == 0 {
+ break
+ }
+ }
+
+ if start < 0 {
+ // no match: skip scoring
+ return -1, 0
+ }
+
+ // Third phase: find the shortest match, and compute the score.
+
+ // Score is the average score for each character.
+ //
+ // A character score is the multiple of:
+ // 1. 1.0 if the character starts a segment, .8 if the character start a
+ // mid-segment word, otherwise 0.6. This carries over to immediately
+ // following characters.
+ // 2. For the final character match, the multiplier from (1) is reduced to
+ // .8 if the next character in the input is a mid-segment word, or 0.6 if
+ // the next character in the input is not a word or segment start. This
+ // ensures that we favor whole-word or whole-segment matches over prefix
+ // matches.
+ // 3. 1.0 if the character is part of the last segment, otherwise
+ // 1.0-.2*<segments from the right>, with a max segment count of 3.
+ //
+ // This is a very naive algorithm, but it is fast. There's lots of prior art
+ // here, and we should leverage it. For example, we could explicitly consider
+ // character distance, and exact matches of words or segments.
+ //
+ // Also note that this might not actually find the highest scoring match, as
+ // doing so could require a non-linear algorithm, depending on how the score
+ // is calculated.
+
+ pi = 0
+ p = m.pattern[pi]
+
+ const (
+ segStreak = 1.0
+ wordStreak = 0.8
+ noStreak = 0.6
+ perSegment = 0.2 // we count at most 3 segments above
+ )
+
+ streakBonus := noStreak
+ totScore := 0.0
+ for ii := uint8(start); ii < inputLen; ii++ {
+ r := m.inputBuffer[ii]
+ if r == p {
+ pi++
+ p = m.pattern[pi]
+ // Note: this could be optimized with some bit operations.
+ switch {
+ case m.roles[ii]&segmentStart != 0 && segStreak > streakBonus:
+ streakBonus = segStreak
+ case m.roles[ii]&wordStart != 0 && wordStreak > streakBonus:
+ streakBonus = wordStreak
+ }
+ finalChar := pi >= m.patternLen
+ // finalCost := 1.0
+ if finalChar && streakBonus > noStreak {
+ switch {
+ case ii == inputLen-1 || m.roles[ii+1]&segmentStart != 0:
+ // Full segment: no reduction
+ case m.roles[ii+1]&wordStart != 0:
+ streakBonus = wordStreak
+ default:
+ streakBonus = noStreak
+ }
+ }
+ totScore += streakBonus * (1.0 - float64(m.segments[ii])*perSegment)
+ if finalChar {
+ break
+ }
+ } else {
+ streakBonus = noStreak
+ }
+ }
+
+ return start, totScore / float64(m.patternLen)
+}
diff --git a/internal/postgres/search.go b/internal/postgres/search.go
index baf1e6c..b561b68 100644
--- a/internal/postgres/search.go
+++ b/internal/postgres/search.go
@@ -94,68 +94,11 @@
"symbol": (*DB).symbolSearch,
}
-// SearchOptions provide information used by db.Search.
-type SearchOptions struct {
- // Maximum number of results to return (page size).
- MaxResults int
+type SearchOptions = internal.SearchOptions
+type SearchResult = internal.SearchResult
- // Offset for DB query.
- Offset int
-
- // Maximum number to use for total result count.
- MaxResultCount int
-
- // If true, perform a symbol search.
- SearchSymbols bool
-
- // SymbolFilter is the word in a search query with a # prefix.
- SymbolFilter string
-}
-
-// SearchResult represents a single search result from SearchDocuments.
-type SearchResult struct {
- Name string
- PackagePath string
- ModulePath string
- Version string
- Synopsis string
- Licenses []string
-
- CommitTime time.Time
-
- // Score is used to sort items in an array of SearchResult.
- Score float64
-
- // NumImportedBy is the number of packages that import PackagePath.
- NumImportedBy uint64
-
- // SameModule is a list of SearchResults from the same module as this one,
- // with lower scores.
- SameModule []*SearchResult
-
- // OtherMajor is a map from module paths with the same series path but at
- // different major versions of this module, to major version.
- // The major version for a non-vN module path (either 0 or 1) is computed
- // based on the version in search documents.
- OtherMajor map[string]int
-
- // NumResults is the total number of packages that were returned for this
- // search.
- NumResults uint64
-
- // Symbol information returned by a search request.
- // Only populated for symbol search mode.
- SymbolName string
- SymbolKind internal.SymbolKind
- SymbolSynopsis string
- SymbolGOOS string
- SymbolGOARCH string
-
- // Offset is the 0-based number of this row in the DB query results, which
- // is the value to use in a SQL OFFSET clause to have this row be the first
- // one returned.
- Offset int
-}
+// SupportsSearch implements the DataSource interface, returning true.
+func (db *DB) SupportsSearch() bool { return true }
// Search executes two search requests concurrently:
// - a sequential scan of packages in descending order of popularity.
diff --git a/internal/testing/testhelper/testhelper.go b/internal/testing/testhelper/testhelper.go
index 52e7003..3ccaef9 100644
--- a/internal/testing/testhelper/testhelper.go
+++ b/internal/testing/testhelper/testhelper.go
@@ -23,6 +23,7 @@
"github.com/google/go-cmp/cmp"
"golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/tools/txtar"
)
const (
@@ -120,28 +121,56 @@
if err != nil {
return "", err
}
+ if err := writeTestDirectory(tempDir, files); err != nil {
+ return "", err
+ }
+ return tempDir, nil
+}
+
+func writeTestDirectory(tempDir string, files map[string]string) error {
for path, contents := range files {
path = filepath.Join(tempDir, path)
parent, _ := filepath.Split(path)
if err := os.MkdirAll(parent, 0755); err != nil {
- return "", err
+ return err
}
file, err := os.Create(path)
if err != nil {
- return "", err
+ return err
}
if _, err := file.WriteString(contents); err != nil {
- return "", err
+ return err
}
if err := file.Close(); err != nil {
- return "", err
+ return err
}
}
+ return nil
+}
- return tempDir, nil
+// WriteTxtarToTempDir parses data as a txtar archive, and writes the resulting
+// files to a new tempdir created with t.TempDir(). It returns the temp
+// directory and files that were unpacked.
+func WriteTxtarToTempDir(t *testing.T, data string) (string, map[string]string) {
+ t.Helper()
+
+ archive := txtar.Parse([]byte(data))
+ files := make(map[string]string)
+ for _, file := range archive.Files {
+ if _, ok := files[file.Name]; ok {
+ t.Fatalf("file %q occurs twice in the provided archive", file.Name)
+ }
+ files[file.Name] = string(file.Data)
+ }
+
+ tempDir := t.TempDir()
+ if err := writeTestDirectory(tempDir, files); err != nil {
+ t.Fatalf("writing test dir: %v", err)
+ }
+ return tempDir, files
}
func CompareWithGolden(t *testing.T, got, filename string, update bool) {
