internal/datasource: combine datasource packages
The proxy and local datasource implementations can share a lot of code.
Put them in the same package to facilitate that.
For golang/go#47780
Change-Id: Ic67d647decd7056650120d47fba9f2351426142b
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/344530
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index 9473985..2fbd6a7 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -18,13 +18,13 @@
"golang.org/x/pkgsite/cmd/internal/cmdconfig"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/datasource"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/frontend"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/middleware"
"golang.org/x/pkgsite/internal/postgres"
"golang.org/x/pkgsite/internal/proxy"
- "golang.org/x/pkgsite/internal/proxydatasource"
"golang.org/x/pkgsite/internal/queue"
"golang.org/x/pkgsite/internal/source"
)
@@ -76,11 +76,11 @@
}
if *directProxy {
- var pds *proxydatasource.DataSource
+ var pds *datasource.ProxyDataSource
if *bypassLicenseCheck {
- pds = proxydatasource.NewBypassingLicenseCheck(proxyClient)
+ pds = datasource.NewBypassingLicenseCheck(proxyClient)
} else {
- pds = proxydatasource.New(proxyClient)
+ pds = datasource.NewProxy(proxyClient)
}
dsg = func(context.Context) internal.DataSource { return pds }
} else {
diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index 6d3cb58..1d82356 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -29,10 +29,10 @@
"github.com/google/safehtml/template"
"golang.org/x/pkgsite/internal"
+ "golang.org/x/pkgsite/internal/datasource"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/fetch"
"golang.org/x/pkgsite/internal/frontend"
- "golang.org/x/pkgsite/internal/localdatasource"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/middleware"
"golang.org/x/pkgsite/internal/source"
@@ -67,7 +67,7 @@
}
func newServer(ctx context.Context, paths []string, gopathMode bool) (*frontend.Server, error) {
- lds := localdatasource.New(source.NewClient(time.Second))
+ lds := datasource.NewLocal(source.NewClient(time.Second))
server, err := frontend.NewServer(frontend.ServerConfig{
DataSourceGetter: func(context.Context) internal.DataSource { return lds },
StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
@@ -80,7 +80,7 @@
}
// load loads local modules from pathList.
-func addGetters(ctx context.Context, ds *localdatasource.DataSource, paths []string, gopathMode bool) {
+func addGetters(ctx context.Context, ds *datasource.LocalDataSource, paths []string, gopathMode bool) {
loaded := len(paths)
for _, path := range paths {
var (
@@ -88,7 +88,7 @@
err error
)
if gopathMode {
- mg, err = localdatasource.NewGOPATHModuleGetter(path)
+ mg, err = datasource.NewGOPATHModuleGetter(path)
} else {
mg, err = fetch.NewDirectoryModuleGetter("", path)
}
diff --git a/internal/localdatasource/datasource.go b/internal/datasource/local.go
similarity index 76%
rename from internal/localdatasource/datasource.go
rename to internal/datasource/local.go
index 6cb1e6b..da0eada 100644
--- a/internal/localdatasource/datasource.go
+++ b/internal/datasource/local.go
@@ -2,10 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package localdatasource implements an in-memory internal.DataSource used to load
-// and display documentation for local modules that are not available via a proxy.
-// Similar to proxydatasource, search and other tabs are not supported in this mode.
-package localdatasource
+package datasource
import (
"context"
@@ -24,9 +21,9 @@
"golang.org/x/pkgsite/internal/source"
)
-// DataSource implements an in-memory internal.DataSource used to display documentation
-// locally. DataSource is not backed by a database or a proxy instance.
-type DataSource struct {
+// LocalDataSource implements an in-memory internal.DataSource used to display documentation
+// locally. It is not backed by a database or a proxy instance.
+type LocalDataSource struct {
sourceClient *source.Client
mu sync.Mutex
@@ -36,8 +33,8 @@
// New creates and returns a new local datasource that bypasses license
// checks by default.
-func New(sc *source.Client) *DataSource {
- return &DataSource{
+func NewLocal(sc *source.Client) *LocalDataSource {
+ return &LocalDataSource{
sourceClient: sc,
loadedModules: make(map[string]*internal.Module),
}
@@ -46,7 +43,7 @@
// AddModuleGetter adds a module getter to the DataSource. To look up a module,
// the getters are tried in the order they were added until the desired module
// is found.
-func (ds *DataSource) AddModuleGetter(g fetch.ModuleGetter) {
+func (ds *LocalDataSource) AddModuleGetter(g fetch.ModuleGetter) {
ds.mu.Lock()
defer ds.mu.Unlock()
ds.getters = append(ds.getters, g)
@@ -54,7 +51,7 @@
// getModule gets the module at the given path and version. It first checks the
// cache, and if it isn't there it then tries to fetch it.
-func (ds *DataSource) getModule(ctx context.Context, path, version string) (*internal.Module, error) {
+func (ds *LocalDataSource) getModule(ctx context.Context, path, version string) (*internal.Module, error) {
if m := ds.getFromCache(path, version); m != nil {
return m, nil
}
@@ -69,7 +66,7 @@
}
// getFromCache returns a module from the cache if it is present, and nil otherwise.
-func (ds *DataSource) getFromCache(path, version string) *internal.Module {
+func (ds *LocalDataSource) getFromCache(path, version string) *internal.Module {
ds.mu.Lock()
defer ds.mu.Unlock()
// Look for an exact match first.
@@ -82,7 +79,7 @@
// fetch fetches a module using the configured ModuleGetters.
// It tries each getter in turn until it finds one that has the module.
-func (ds *DataSource) fetch(ctx context.Context, modulePath, version string) (_ *internal.Module, err error) {
+func (ds *LocalDataSource) fetch(ctx context.Context, modulePath, version string) (_ *internal.Module, err error) {
log.Infof(ctx, "local DataSource: fetching %s@%s", modulePath, version)
start := time.Now()
defer func() {
@@ -145,7 +142,7 @@
// GetUnit returns information about a unit. Both the module path and package
// path must be known.
-func (ds *DataSource) GetUnit(ctx context.Context, pathInfo *internal.UnitMeta, fields internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) {
+func (ds *LocalDataSource) GetUnit(ctx context.Context, pathInfo *internal.UnitMeta, fields internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) {
defer derrors.Wrap(&err, "GetUnit(%q, %q)", pathInfo.Path, pathInfo.ModulePath)
module, err := ds.getModule(ctx, pathInfo.ModulePath, pathInfo.Version)
@@ -162,7 +159,7 @@
}
// GetUnitMeta returns information about a path.
-func (ds *DataSource) GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *internal.UnitMeta, err error) {
+func (ds *LocalDataSource) GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *internal.UnitMeta, err error) {
defer derrors.Wrap(&err, "GetUnitMeta(%q, %q, %q)", path, requestedModulePath, requestedVersion)
module, err := ds.findModule(ctx, path, requestedModulePath, requestedVersion)
@@ -186,7 +183,7 @@
// findModule finds the module with longest module path containing the given
// package path. It returns an error if no module is found.
-func (ds *DataSource) findModule(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.Module, err error) {
+func (ds *LocalDataSource) findModule(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.Module, err error) {
defer derrors.Wrap(&err, "findModule(%q, %q, %q)", pkgPath, modulePath, version)
if modulePath != internal.UnknownModulePath {
@@ -206,16 +203,16 @@
}
// GetLatestInfo is not implemented.
-func (ds *DataSource) GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *internal.UnitMeta) (internal.LatestInfo, error) {
+func (ds *LocalDataSource) GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *internal.UnitMeta) (internal.LatestInfo, error) {
return internal.LatestInfo{}, nil
}
// GetNestedModules is not implemented.
-func (ds *DataSource) GetNestedModules(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error) {
+func (ds *LocalDataSource) GetNestedModules(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error) {
return nil, nil
}
// GetModuleReadme is not implemented.
-func (*DataSource) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*internal.Readme, error) {
+func (*LocalDataSource) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*internal.Readme, error) {
return nil, nil
}
diff --git a/internal/localdatasource/datasource_test.go b/internal/datasource/local_test.go
similarity index 91%
rename from internal/localdatasource/datasource_test.go
rename to internal/datasource/local_test.go
index 0a96178..f1baab0 100644
--- a/internal/localdatasource/datasource_test.go
+++ b/internal/datasource/local_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package localdatasource
+package datasource
import (
"context"
@@ -13,25 +13,17 @@
"time"
"github.com/google/go-cmp/cmp"
- "github.com/google/safehtml/template"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/fetch"
- "golang.org/x/pkgsite/internal/godoc/dochtml"
- "golang.org/x/pkgsite/internal/licenses"
"golang.org/x/pkgsite/internal/source"
"golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/testing/testhelper"
)
-func TestMain(m *testing.M) {
- os.Exit(run(m))
-}
+var datasource *LocalDataSource
-var datasource *DataSource
-
-func run(m *testing.M) int {
- licenses.OmitExceptions = true
+func setupLocal() func() {
modules := []map[string]string{
{
"go.mod": "module github.com/my/module\n\ngo 1.12",
@@ -77,25 +69,28 @@
},
}
- dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../static/doc"))
- datasource = New(source.NewClientForTesting())
+ datasource = NewLocal(source.NewClientForTesting())
+ var dirs []string
for _, module := range modules {
directory, err := testhelper.CreateTestDirectory(module)
if err != nil {
log.Fatal(err)
}
- defer os.RemoveAll(directory)
-
+ dirs = append(dirs, directory)
mg, err := fetch.NewDirectoryModuleGetter("", directory)
if err != nil {
log.Fatal(err)
}
datasource.AddModuleGetter(mg)
}
- return m.Run()
+ return func() {
+ for _, d := range dirs {
+ os.RemoveAll(d)
+ }
+ }
}
-func TestGetUnitMeta(t *testing.T) {
+func TestLocalGetUnitMeta(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
@@ -202,7 +197,7 @@
}
}
-func TestGetUnit(t *testing.T) {
+func TestLocalGetUnit(t *testing.T) {
// This is a simple test to verify that data is fetched correctly. The
// return value of FetchResult is tested in internal/fetch so no need
// to repeat it.
diff --git a/internal/proxydatasource/datasource.go b/internal/datasource/proxy.go
similarity index 81%
rename from internal/proxydatasource/datasource.go
rename to internal/datasource/proxy.go
index 43ba301..f876233 100644
--- a/internal/proxydatasource/datasource.go
+++ b/internal/datasource/proxy.go
@@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package proxydatasource implements an internal.DataSource backed solely by a
-// proxy instance.
-package proxydatasource
+// Package datasource provides internal.DataSource implementations backed solely
+// by a proxy instance, and backed by the local filesystem.
+// Search and other tabs are not supported by these implementations.
+package datasource
import (
"context"
@@ -25,19 +26,19 @@
"golang.org/x/pkgsite/internal/version"
)
-var _ internal.DataSource = (*DataSource)(nil)
+var _ internal.DataSource = (*ProxyDataSource)(nil)
// New returns a new direct proxy datasource.
-func New(proxyClient *proxy.Client) *DataSource {
- return newDataSource(proxyClient, source.NewClient(1*time.Minute))
+func NewProxy(proxyClient *proxy.Client) *ProxyDataSource {
+ return newProxyDataSource(proxyClient, source.NewClient(1*time.Minute))
}
-func NewForTesting(proxyClient *proxy.Client) *DataSource {
- return newDataSource(proxyClient, source.NewClientForTesting())
+func NewForTesting(proxyClient *proxy.Client) *ProxyDataSource {
+ return newProxyDataSource(proxyClient, source.NewClientForTesting())
}
-func newDataSource(proxyClient *proxy.Client, sourceClient *source.Client) *DataSource {
- return &DataSource{
+func newProxyDataSource(proxyClient *proxy.Client, sourceClient *source.Client) *ProxyDataSource {
+ return &ProxyDataSource{
proxyClient: proxyClient,
sourceClient: sourceClient,
versionCache: make(map[versionKey]*versionEntry),
@@ -50,15 +51,15 @@
// NewBypassingLicenseCheck returns a new direct proxy datasource that bypasses
// license checks. That means all data will be returned for non-redistributable
// modules, packages and directories.
-func NewBypassingLicenseCheck(c *proxy.Client) *DataSource {
- ds := New(c)
+func NewBypassingLicenseCheck(c *proxy.Client) *ProxyDataSource {
+ ds := NewProxy(c)
ds.bypassLicenseCheck = true
return ds
}
-// DataSource implements the frontend.DataSource interface, by querying a
+// ProxyDataSource implements the frontend.DataSource interface, by querying a
// module proxy directly and caching the results in memory.
-type DataSource struct {
+type ProxyDataSource struct {
proxyClient *proxy.Client
sourceClient *source.Client
@@ -86,7 +87,7 @@
// getModule retrieves a version from the cache, or failing that queries and
// processes the version from the proxy.
-func (ds *DataSource) getModule(ctx context.Context, modulePath, version string, _ internal.BuildContext) (_ *internal.Module, err error) {
+func (ds *ProxyDataSource) getModule(ctx context.Context, modulePath, version string, _ internal.BuildContext) (_ *internal.Module, err error) {
defer derrors.Wrap(&err, "getModule(%q, %q)", modulePath, version)
key := versionKey{modulePath, version}
@@ -163,7 +164,7 @@
// using the given finder func and iteratively testing parent directories of
// the import path. It performs no testing as to whether the specified module
// version that was found actually contains a package corresponding to pkgPath.
-func (ds *DataSource) findModule(ctx context.Context, pkgPath string, version string) (_ string, _ *proxy.VersionInfo, err error) {
+func (ds *ProxyDataSource) findModule(ctx context.Context, pkgPath string, version string) (_ string, _ *proxy.VersionInfo, err error) {
defer derrors.Wrap(&err, "findModule(%q, ...)", pkgPath)
pkgPath = strings.TrimLeft(pkgPath, "/")
for _, modulePath := range internal.CandidateModulePaths(pkgPath) {
@@ -180,7 +181,7 @@
}
// getUnit returns information about a unit.
-func (ds *DataSource) getUnit(ctx context.Context, fullPath, modulePath, version string, bc internal.BuildContext) (_ *internal.Unit, err error) {
+func (ds *ProxyDataSource) getUnit(ctx context.Context, fullPath, modulePath, version string, bc internal.BuildContext) (_ *internal.Unit, err error) {
var m *internal.Module
m, err = ds.getModule(ctx, modulePath, version, bc)
if err != nil {
@@ -195,7 +196,7 @@
}
// GetLatestInfo returns latest information for unitPath and modulePath.
-func (ds *DataSource) GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *internal.UnitMeta) (latest internal.LatestInfo, err error) {
+func (ds *ProxyDataSource) GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *internal.UnitMeta) (latest internal.LatestInfo, err error) {
defer derrors.Wrap(&err, "GetLatestInfo(ctx, %q, %q)", unitPath, modulePath)
if latestUnitMeta == nil {
@@ -220,7 +221,7 @@
// of the latest version found in the proxy by iterating through vN versions.
// This function does not attempt to find whether the full path exists
// in the new major version.
-func (ds *DataSource) getLatestMajorVersion(ctx context.Context, fullPath, modulePath string) (_ string, _ string, err error) {
+func (ds *ProxyDataSource) getLatestMajorVersion(ctx context.Context, fullPath, modulePath string) (_ string, _ string, err error) {
// We are checking if the full path is valid so that we can forward the error if not.
seriesPath := internal.SeriesPathForModule(modulePath)
info, err := ds.proxyClient.Info(ctx, seriesPath, version.Latest)
diff --git a/internal/proxydatasource/details.go b/internal/datasource/proxy_details.go
similarity index 68%
rename from internal/proxydatasource/details.go
rename to internal/datasource/proxy_details.go
index 71481e2..1bf07db 100644
--- a/internal/proxydatasource/details.go
+++ b/internal/datasource/proxy_details.go
@@ -2,9 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package proxydatasource implements an internal.DataSource backed solely by a
-// proxy instance.
-package proxydatasource
+package datasource
import (
"context"
@@ -15,14 +13,14 @@
)
// GetUnit returns information about a directory at a path.
-func (ds *DataSource) GetUnit(ctx context.Context, um *internal.UnitMeta, field internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) {
+func (ds *ProxyDataSource) GetUnit(ctx context.Context, um *internal.UnitMeta, field internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) {
defer derrors.Wrap(&err, "GetUnit(%q, %q, %q)", um.Path, um.ModulePath, um.Version)
return ds.getUnit(ctx, um.Path, um.ModulePath, um.Version, bc)
}
// GetModuleInfo returns the ModuleInfo as fetched from the proxy for module
// version specified by modulePath and version.
-func (ds *DataSource) GetModuleInfo(ctx context.Context, modulePath, version string) (_ *internal.ModuleInfo, err error) {
+func (ds *ProxyDataSource) GetModuleInfo(ctx context.Context, modulePath, version string) (_ *internal.ModuleInfo, err error) {
defer derrors.Wrap(&err, "GetModuleInfo(%q, %q)", modulePath, version)
m, err := ds.getModule(ctx, modulePath, version, internal.BuildContext{})
if err != nil {
@@ -32,7 +30,7 @@
}
// GetUnitMeta returns information about the given path.
-func (ds *DataSource) GetUnitMeta(ctx context.Context, path, inModulePath, inVersion string) (_ *internal.UnitMeta, err error) {
+func (ds *ProxyDataSource) GetUnitMeta(ctx context.Context, path, inModulePath, inVersion string) (_ *internal.UnitMeta, err error) {
defer derrors.Wrap(&err, "GetUnitMeta(%q, %q, %q)", path, inModulePath, inVersion)
var info *proxy.VersionInfo
@@ -66,16 +64,16 @@
}
// GetExperiments is unimplemented.
-func (*DataSource) GetExperiments(ctx context.Context) ([]*internal.Experiment, error) {
+func (*ProxyDataSource) GetExperiments(ctx context.Context) ([]*internal.Experiment, error) {
return nil, nil
}
// GetNestedModules will return an empty slice since it is not implemented in proxy mode.
-func (ds *DataSource) GetNestedModules(ctx context.Context, modulePath string) (_ []*internal.ModuleInfo, err error) {
+func (ds *ProxyDataSource) GetNestedModules(ctx context.Context, modulePath string) (_ []*internal.ModuleInfo, err error) {
return nil, nil
}
// GetModuleReadme is unimplemented.
-func (ds *DataSource) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*internal.Readme, error) {
+func (ds *ProxyDataSource) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*internal.Readme, error) {
return nil, nil
}
diff --git a/internal/proxydatasource/datasource_test.go b/internal/datasource/proxy_test.go
similarity index 97%
rename from internal/proxydatasource/datasource_test.go
rename to internal/datasource/proxy_test.go
index 9c30b30..c16e3d5 100644
--- a/internal/proxydatasource/datasource_test.go
+++ b/internal/datasource/proxy_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package proxydatasource
+package datasource
import (
"context"
@@ -30,10 +30,11 @@
dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../static/doc"))
testModules = proxytest.LoadTestModules("../proxy/testdata")
licenses.OmitExceptions = true
+ defer setupLocal()()
os.Exit(m.Run())
}
-func setup(t *testing.T) (context.Context, *DataSource, func()) {
+func setup(t *testing.T) (context.Context, *ProxyDataSource, func()) {
t.Helper()
client, teardownProxy := proxytest.SetupTestClient(t, testModules)
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
@@ -123,7 +124,7 @@
}
-func TestGetUnitMeta(t *testing.T) {
+func TestProxyGetUnitMeta(t *testing.T) {
ctx, ds, teardown := setup(t)
defer teardown()
