internal/localdatasource: implement a local datasource
Create localdatasource package which implements an in-memory
internal.DataSource to display documentation locally.
Add tests for localdatasource.DataSource.
Updates golang/go#40159
Change-Id: Ie18dd68e6108cfa361c4db31030679fc55661d35
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/260778
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/internal/localdatasource/datasource.go b/internal/localdatasource/datasource.go
new file mode 100644
index 0000000..c08261e
--- /dev/null
+++ b/internal/localdatasource/datasource.go
@@ -0,0 +1,186 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package localdatasource implements an in-memory internal.DataSource used to load
+// and display documentation for local modules that are not available via a proxy.
+// Similar to proxydatasource, search and other tabs are not supported in this mode.
+package localdatasource
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "golang.org/x/pkgsite/internal"
+ "golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/fetch"
+ "golang.org/x/pkgsite/internal/source"
+)
+
+// DataSource implements an in-memory internal.DataSource used to display documentation
+// locally. DataSource is not backed by a database or a proxy instance.
+type DataSource struct {
+ sourceClient *source.Client
+
+ mu sync.Mutex
+ loadedModules map[string]*internal.Module
+}
+
+// New creates and returns a new local datasource that bypasses license
+// checks by default.
+func New() *DataSource {
+ return &DataSource{
+ loadedModules: make(map[string]*internal.Module),
+ }
+}
+
+// Load loads a module from the given local path. Loading is required before
+// being able to display the module.
+func (ds *DataSource) Load(ctx context.Context, localPath string) (err error) {
+ defer derrors.Wrap(&err, "Load(%q)", localPath)
+ return ds.fetch(ctx, "", localPath)
+}
+
+// LoadFromGOPATH loads a module from GOPATH using the given import path. The full
+// path of the module should be GOPATH/src/importPath. If several GOPATHs exist, the
+// module is loaded from the first one that contains the import path. Loading is required
+// before being able to display the module.
+func (ds *DataSource) LoadFromGOPATH(ctx context.Context, importPath string) (err error) {
+ defer derrors.Wrap(&err, "LoadFromGOPATH(%q)", importPath)
+
+ path := getFullPath(importPath)
+ if path == "" {
+ return fmt.Errorf("path %s doesn't exist: %w", importPath, derrors.NotFound)
+ }
+
+ return ds.fetch(ctx, importPath, path)
+}
+
+// fetch fetches a module using FetchLocalModule and adds it to the datasource.
+// If the fetching fails, an error is returned.
+func (ds *DataSource) fetch(ctx context.Context, modulePath, localPath string) error {
+ fr := fetch.FetchLocalModule(ctx, modulePath, localPath, ds.sourceClient)
+ if fr.Error != nil {
+ return fr.Error
+ }
+
+ fr.Module.IsRedistributable = true
+ for _, unit := range fr.Module.Units {
+ unit.IsRedistributable = true
+ }
+ for _, pkg := range fr.Module.LegacyPackages {
+ pkg.IsRedistributable = true
+ }
+
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+ ds.loadedModules[fr.ModulePath] = fr.Module
+ return nil
+}
+
+// getFullPath takes an import path, tests it relative to each GOPATH, and returns
+// a full path to the module. If the given import path doesn't exist in any GOPATH,
+// an empty string is returned.
+func getFullPath(modulePath string) string {
+ gopaths := filepath.SplitList(os.Getenv("GOPATH"))
+ for _, gopath := range gopaths {
+ path := filepath.Join(gopath, "src", modulePath)
+ info, err := os.Stat(path)
+ if err == nil && info.IsDir() {
+ return path
+ }
+ }
+ return ""
+}
+
+// GetUnit returns information about a unit. Both the module path and package
+// path must both be known.
+func (ds *DataSource) GetUnit(ctx context.Context, pathInfo *internal.UnitMeta, fields internal.FieldSet) (_ *internal.Unit, err error) {
+ defer derrors.Wrap(&err, "GetUnit(%q, %q)", pathInfo.Path, pathInfo.ModulePath)
+
+ modulepath := pathInfo.ModulePath
+ path := pathInfo.Path
+
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+ if ds.loadedModules[modulepath] == nil {
+ return nil, fmt.Errorf("%s not loaded: %w", modulepath, derrors.NotFound)
+ }
+
+ module := ds.loadedModules[modulepath]
+ for _, unit := range module.Units {
+ if unit.Path == path {
+ return unit, nil
+ }
+ }
+
+ return nil, fmt.Errorf("%s not found: %w", path, derrors.NotFound)
+}
+
+// GetUnitMeta returns information about a path.
+func (ds *DataSource) GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *internal.UnitMeta, err error) {
+ defer derrors.Wrap(&err, "GetUnitMeta(%q, %q, %q)", path, requestedModulePath, requestedVersion)
+
+ if requestedModulePath == internal.UnknownModulePath {
+ requestedModulePath, err = ds.findModule(path)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ ds.mu.Lock()
+ module := ds.loadedModules[requestedModulePath]
+ ds.mu.Unlock()
+
+ um := &internal.UnitMeta{
+ Path: path,
+ ModulePath: requestedModulePath,
+ Version: fetch.LocalVersion,
+ CommitTime: fetch.LocalCommitTime,
+ }
+
+ for _, u := range module.Units {
+ if u.Path == path {
+ um.Name = u.Name
+ um.IsRedistributable = u.IsRedistributable
+ }
+ }
+
+ return um, nil
+}
+
+// findModule finds the longest module path in loadedModules containing the given
+// package path. It iteratively checks parent directories to find an import path.
+// Returns an error if no module is found.
+func (ds *DataSource) findModule(pkgPath string) (_ string, err error) {
+ defer derrors.Wrap(&err, "findModule(%q)", pkgPath)
+
+ pkgPath = strings.TrimLeft(pkgPath, "/")
+
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+ for modulePath := pkgPath; modulePath != "" && modulePath != "."; modulePath = path.Dir(modulePath) {
+ if ds.loadedModules[modulePath] != nil {
+ return modulePath, nil
+ }
+ }
+
+ return "", fmt.Errorf("%s not loaded: %w", pkgPath, derrors.NotFound)
+}
+
+// GetLatestMajorVersion returns the latest major version of a series path.
+// When fetching local modules, version is not accounted for, so an empty
+// string is returned.
+func (ds *DataSource) GetLatestMajorVersion(ctx context.Context, seriesPath string) (string, error) {
+ return "", nil
+}
+
+// GetNestedModules is not implemented.
+func (ds *DataSource) GetNestedModules(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error) {
+ return nil, nil
+}
diff --git a/internal/localdatasource/datasource_test.go b/internal/localdatasource/datasource_test.go
new file mode 100644
index 0000000..69a37a8
--- /dev/null
+++ b/internal/localdatasource/datasource_test.go
@@ -0,0 +1,237 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package localdatasource
+
+import (
+ "context"
+ "errors"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite/internal"
+ "golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/fetch"
+ "golang.org/x/pkgsite/internal/testing/testhelper"
+)
+
+var (
+ ctx context.Context
+ cancel func()
+ datasource *DataSource
+)
+
+func setup(t *testing.T) (context.Context, func(), *DataSource, error) {
+ t.Helper()
+
+ // Setup only once.
+ if datasource != nil {
+ return ctx, cancel, datasource, nil
+ }
+
+ modules := []map[string]string{
+ {
+ "go.mod": "module github.com/my/module\n\ngo 1.12",
+ "LICENSE": testhelper.BSD0License,
+ "README.md": "README FILE FOR TESTING.",
+ "bar/COPYING": testhelper.MITLicense,
+ "bar/README.md": "Another README FILE FOR TESTING.",
+ "bar/bar.go": `
+ // package bar
+ package bar
+
+ // Bar returns the string "bar".
+ func Bar() string {
+ return "bar"
+ }`,
+ "foo/LICENSE.md": testhelper.MITLicense,
+ "foo/foo.go": `
+ // package foo
+ package foo
+
+ import (
+ "fmt"
+
+ "github.com/my/module/bar"
+ )
+
+ // FooBar returns the string "foo bar".
+ func FooBar() string {
+ return fmt.Sprintf("foo %s", bar.Bar())
+ }`,
+ },
+ {
+ "go.mod": "module github.com/no/license\n\ngo 1.12",
+ "LICENSE": "unknown",
+ "bar/bar.go": `
+ // package bar
+ package bar
+
+ // Bar returns the string "bar".
+ func Bar() string {
+ return "bar"
+ }`,
+ },
+ }
+
+ datasource = New()
+ ctx, cancel = context.WithTimeout(context.Background(), 20*time.Second)
+ for _, module := range modules {
+ directory, err := testhelper.CreateTestDirectory(module)
+ if err != nil {
+ return ctx, func() { cancel() }, nil, err
+ }
+ defer os.RemoveAll(directory)
+
+ err = datasource.Load(ctx, directory)
+ if err != nil {
+ return ctx, func() { cancel() }, nil, err
+ }
+ }
+
+ return ctx, func() { cancel() }, datasource, nil
+}
+
+func TestGetUnitMeta(t *testing.T) {
+ ctx, cancel, ds, err := setup(t)
+ if err != nil {
+ t.Fatalf("setup failed: %s", err.Error())
+ }
+ defer cancel()
+
+ for _, test := range []struct {
+ path, modulePath string
+ want *internal.UnitMeta
+ wantErr error
+ }{
+ {
+ path: "github.com/my/module",
+ modulePath: "github.com/my/module",
+ want: &internal.UnitMeta{
+ Path: "github.com/my/module",
+ ModulePath: "github.com/my/module",
+ IsRedistributable: true,
+ Version: fetch.LocalVersion,
+ CommitTime: fetch.LocalCommitTime,
+ },
+ },
+ {
+ path: "github.com/my/module/bar",
+ modulePath: "github.com/my/module",
+ want: &internal.UnitMeta{
+ Path: "github.com/my/module/bar",
+ Name: "bar",
+ ModulePath: "github.com/my/module",
+ IsRedistributable: true,
+ Version: fetch.LocalVersion,
+ CommitTime: fetch.LocalCommitTime,
+ },
+ },
+ {
+ path: "github.com/my/module/foo",
+ modulePath: "github.com/my/module",
+ want: &internal.UnitMeta{
+ Path: "github.com/my/module/foo",
+ Name: "foo",
+ ModulePath: "github.com/my/module",
+ IsRedistributable: true,
+ Version: fetch.LocalVersion,
+ CommitTime: fetch.LocalCommitTime,
+ },
+ },
+ {
+ path: "github.com/my/module/bar",
+ modulePath: internal.UnknownModulePath,
+ want: &internal.UnitMeta{
+ Path: "github.com/my/module/bar",
+ Name: "bar",
+ ModulePath: "github.com/my/module",
+ IsRedistributable: true,
+ Version: fetch.LocalVersion,
+ CommitTime: fetch.LocalCommitTime,
+ },
+ },
+ {
+ path: "github.com/not/loaded",
+ modulePath: internal.UnknownModulePath,
+ wantErr: derrors.NotFound,
+ },
+ } {
+ t.Run(test.path, func(t *testing.T) {
+ got, err := ds.GetUnitMeta(ctx, test.path, test.modulePath, fetch.LocalVersion)
+ if test.wantErr != nil {
+ if !errors.Is(err, test.wantErr) {
+ t.Errorf("GetUnitMeta(%q, %q): %v; wantErr = %v)", test.path, test.modulePath, err, test.wantErr)
+ }
+ } else {
+ if err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(test.want, got); diff != "" {
+ t.Errorf("mismatch (-want +got):\n%s", diff)
+
+ }
+ }
+ })
+ }
+}
+
+func TestGetUnit(t *testing.T) {
+ // This is a simple test to verify that data is fetched correctly. The
+ // return value of FetchResult is tested in internal/fetch so no need
+ // to repeat it.
+ ctx, cancel, ds, err := setup(t)
+ if err != nil {
+ t.Fatalf("setup failed: %s", err.Error())
+ }
+ defer cancel()
+
+ for _, test := range []struct {
+ path, modulePath string
+ wantLoaded bool
+ }{
+ {
+ path: "github.com/my/module",
+ modulePath: "github.com/my/module",
+ wantLoaded: true,
+ },
+ {
+ path: "github.com/my/module/foo",
+ modulePath: "github.com/my/module",
+ wantLoaded: true,
+ },
+ {
+ path: "github.com/no/license/bar",
+ modulePath: "github.com/no/license",
+ wantLoaded: true,
+ },
+ {
+ path: "github.com/not/loaded",
+ modulePath: internal.UnknownModulePath,
+ },
+ } {
+ t.Run(test.path, func(t *testing.T) {
+ um := &internal.UnitMeta{
+ Path: test.path,
+ ModulePath: test.modulePath,
+ }
+ got, err := ds.GetUnit(ctx, um, 0)
+ if !test.wantLoaded {
+ if err == nil {
+ t.Fatalf("returned not loaded module %q", test.path)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("failed for %q: %q", test.path, err.Error())
+ }
+
+ if gotEmpty := (got.Documentation == nil && got.Readme == nil); gotEmpty {
+ t.Errorf("%q: gotEmpty = %t", test.path, gotEmpty)
+ }
+ })
+ }
+}
