internal/fetch: move all getters to a single source file
Pure code motion.
For golang/go#47834
Change-Id: Ieef2491431fa37b68a184eb9e8e309d3437a8f48
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/343953
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go
index e7847eb..c9fe09b 100644
--- a/internal/fetch/fetch.go
+++ b/internal/fetch/fetch.go
@@ -90,20 +90,6 @@
}
)
-// ModuleGetter gets module data.
-type ModuleGetter interface {
- // Info returns basic information about the module.
- Info(ctx context.Context, path, version string) (*proxy.VersionInfo, error)
- // Mod returns the contents of the module's go.mod file.
- Mod(ctx context.Context, path, version string) ([]byte, error)
- // FS returns an FS for the module's contents. The FS should match the format
- // of a module zip file.
- FS(ctx context.Context, path, version string) (fs.FS, error)
- // ZipSize returns the approximate size of the zip file in bytes.
- // It is used only for load-shedding.
- ZipSize(ctx context.Context, path, version string) (int64, error)
-}
-
type FetchResult struct {
ModulePath string
RequestedVersion string
diff --git a/internal/fetch/fetchlocal.go b/internal/fetch/fetchlocal.go
deleted file mode 100644
index b7ffcbd..0000000
--- a/internal/fetch/fetchlocal.go
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package fetch
-
-import (
- "archive/zip"
- "bytes"
- "context"
- "errors"
- "fmt"
- "io"
- "io/fs"
- "io/ioutil"
- "os"
- "path/filepath"
- "strings"
- "time"
-
- "golang.org/x/mod/modfile"
- "golang.org/x/pkgsite/internal/derrors"
- "golang.org/x/pkgsite/internal/proxy"
-)
-
-// Version and commit time are pre specified when fetching a local module, as these
-// fields are normally obtained from a proxy.
-var (
- LocalVersion = "v0.0.0"
- LocalCommitTime = time.Time{}
-)
-
-// A directoryModuleGetter is a ModuleGetter whose source is a directory in the file system that contains
-// a module's files.
-type directoryModuleGetter struct {
- modulePath string
- dir string
-}
-
-// NewDirectoryModuleGetter returns a ModuleGetter for reading a module from a directory.
-func NewDirectoryModuleGetter(modulePath, dir string) (*directoryModuleGetter, error) {
- if modulePath == "" {
- goModBytes, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
- if err != nil {
- return nil, fmt.Errorf("cannot obtain module path for %q (%v): %w", dir, err, derrors.BadModule)
- }
- modulePath = modfile.ModulePath(goModBytes)
- if modulePath == "" {
- return nil, fmt.Errorf("go.mod in %q has no module path: %w", dir, derrors.BadModule)
- }
- }
- return &directoryModuleGetter{
- dir: dir,
- modulePath: modulePath,
- }, nil
-}
-
-func (g *directoryModuleGetter) checkPath(path string) error {
- if path != g.modulePath {
- return fmt.Errorf("given module path %q does not match %q for directory %q: %w",
- path, g.modulePath, g.dir, derrors.NotFound)
- }
- return nil
-}
-
-// Info returns basic information about the module.
-func (g *directoryModuleGetter) Info(ctx context.Context, path, version string) (*proxy.VersionInfo, error) {
- if err := g.checkPath(path); err != nil {
- return nil, err
- }
- return &proxy.VersionInfo{
- Version: LocalVersion,
- Time: LocalCommitTime,
- }, nil
-}
-
-// Mod returns the contents of the module's go.mod file.
-// If the file does not exist, it returns a synthesized one.
-func (g *directoryModuleGetter) Mod(ctx context.Context, path, version string) ([]byte, error) {
- if err := g.checkPath(path); err != nil {
- return nil, err
- }
- data, err := ioutil.ReadFile(filepath.Join(g.dir, "go.mod"))
- if errors.Is(err, os.ErrNotExist) {
- return []byte(fmt.Sprintf("module %s\n", g.modulePath)), nil
- }
- return data, err
-}
-
-// FS returns an fs.FS for the module.
-func (g *directoryModuleGetter) FS(ctx context.Context, path, version string) (fs.FS, error) {
- if err := g.checkPath(path); err != nil {
- return nil, err
- }
- return createZipReader(g.dir, path, LocalVersion)
-}
-
-// ZipSize returns the approximate size of the zip file in bytes.
-func (g *directoryModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
- return 0, errors.New("directoryModuleGetter.ZipSize unimplemented")
-}
-
-// createZipReader creates a zip file from a directory given a local path and
-// returns a zip.Reader to be passed to processZipFile. The purpose of the
-// function is to transform a local go module into a zip file to be processed by
-// existing functions.
-func createZipReader(localPath, modulePath, version string) (*zip.Reader, error) {
- buf := new(bytes.Buffer)
- w := zip.NewWriter(buf)
- err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if info.IsDir() {
- return nil
- }
-
- readFrom, err := os.Open(path)
- if err != nil {
- return err
- }
- defer readFrom.Close()
-
- writeTo, err := w.Create(filepath.Join(moduleVersionDir(modulePath, version), strings.TrimPrefix(path, localPath)))
- if err != nil {
- return err
- }
-
- _, err = io.Copy(writeTo, readFrom)
- return err
- })
- if err != nil {
- return nil, err
- }
- if err := w.Close(); err != nil {
- return nil, err
- }
-
- reader := bytes.NewReader(buf.Bytes())
- return zip.NewReader(reader, reader.Size())
-}
diff --git a/internal/fetch/fs.go b/internal/fetch/fs.go
deleted file mode 100644
index 6a78897..0000000
--- a/internal/fetch/fs.go
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package fetch
-
-import (
- "archive/zip"
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io/fs"
- "io/ioutil"
- "os"
- "path/filepath"
-
- "golang.org/x/mod/module"
- "golang.org/x/pkgsite/internal/derrors"
- "golang.org/x/pkgsite/internal/proxy"
-)
-
-// An fsModuleGetter gets modules from a directory in the filesystem
-// that is organized like the proxy, with paths that correspond to proxy
-// URLs. An example of such a directory is $(go env GOMODCACHE)/cache/download.
-type fsModuleGetter struct {
- dir string
-}
-
-// NewFSModuleGetter return a ModuleGetter that reads modules from a filesystem
-// directory organized like the proxy.
-func NewFSModuleGetter(dir string) ModuleGetter {
- return &fsModuleGetter{dir: dir}
-}
-
-// Info returns basic information about the module.
-func (g *fsModuleGetter) Info(ctx context.Context, path, version string) (_ *proxy.VersionInfo, err error) {
- defer derrors.Wrap(&err, "fsModuleGetter.Info(%q, %q)", path, version)
-
- data, err := g.readFile(path, version, "info")
- if err != nil {
- return nil, err
- }
- var info proxy.VersionInfo
- if err := json.Unmarshal(data, &info); err != nil {
- return nil, err
- }
- return &info, nil
-}
-
-// Mod returns the contents of the module's go.mod file.
-func (g *fsModuleGetter) Mod(ctx context.Context, path, version string) (_ []byte, err error) {
- defer derrors.Wrap(&err, "fsModuleGetter.Mod(%q, %q)", path, version)
-
- return g.readFile(path, version, "mod")
-}
-
-// FS returns an FS for the module's zip file.
-func (g *fsModuleGetter) FS(ctx context.Context, path, version string) (_ fs.FS, err error) {
- defer derrors.Wrap(&err, "fsModuleGetter.FS(%q, %q)", path, version)
-
- data, err := g.readFile(path, version, "zip")
- if err != nil {
- return nil, err
- }
- return zip.NewReader(bytes.NewReader(data), int64(len(data)))
-}
-
-// ZipSize returns the approximate size of the zip file in bytes.
-func (g *fsModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
- return 0, errors.New("fsModuleGetter.ZipSize unimplemented")
-}
-
-func (g *fsModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
- epath, err := g.escapedPath(path, version, suffix)
- if err != nil {
- return nil, err
- }
- f, err := os.Open(epath)
- if err != nil {
- return nil, err
- }
- defer f.Close()
- return ioutil.ReadAll(f)
-}
-
-func (g *fsModuleGetter) escapedPath(modulePath, version, suffix string) (string, error) {
- ep, err := module.EscapePath(modulePath)
- if err != nil {
- return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
- }
- ev, err := module.EscapeVersion(version)
- if err != nil {
- return "", fmt.Errorf("version: %v: %w", err, derrors.InvalidArgument)
- }
- return filepath.Join(g.dir, ep, "@v", fmt.Sprintf("%s.%s", ev, suffix)), nil
-}
diff --git a/internal/fetch/getters.go b/internal/fetch/getters.go
index 8b05250..89591cd 100644
--- a/internal/fetch/getters.go
+++ b/internal/fetch/getters.go
@@ -4,13 +4,43 @@
package fetch
-import (
- "context"
- "io/fs"
+// The ModuleGetter interface and its implementations.
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "golang.org/x/mod/modfile"
+ "golang.org/x/mod/module"
+ "golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/proxy"
)
+// ModuleGetter gets module data.
+type ModuleGetter interface {
+ // Info returns basic information about the module.
+ Info(ctx context.Context, path, version string) (*proxy.VersionInfo, error)
+ // Mod returns the contents of the module's go.mod file.
+ Mod(ctx context.Context, path, version string) ([]byte, error)
+ // FS returns an FS for the module's contents. The FS should match the format
+ // of a module zip file.
+ FS(ctx context.Context, path, version string) (fs.FS, error)
+ // ZipSize returns the approximate size of the zip file in bytes.
+ // It is used only for load-shedding.
+ ZipSize(ctx context.Context, path, version string) (int64, error)
+}
+
type proxyModuleGetter struct {
prox *proxy.Client
}
@@ -40,3 +70,196 @@
func (g *proxyModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
return g.prox.ZipSize(ctx, path, version)
}
+
+// Version and commit time are pre specified when fetching a local module, as these
+// fields are normally obtained from a proxy.
+var (
+ LocalVersion = "v0.0.0"
+ LocalCommitTime = time.Time{}
+)
+
+// A directoryModuleGetter is a ModuleGetter whose source is a directory in the file system that contains
+// a module's files.
+type directoryModuleGetter struct {
+ modulePath string
+ dir string
+}
+
+// NewDirectoryModuleGetter returns a ModuleGetter for reading a module from a directory.
+func NewDirectoryModuleGetter(modulePath, dir string) (*directoryModuleGetter, error) {
+ if modulePath == "" {
+ goModBytes, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
+ if err != nil {
+ return nil, fmt.Errorf("cannot obtain module path for %q (%v): %w", dir, err, derrors.BadModule)
+ }
+ modulePath = modfile.ModulePath(goModBytes)
+ if modulePath == "" {
+ return nil, fmt.Errorf("go.mod in %q has no module path: %w", dir, derrors.BadModule)
+ }
+ }
+ return &directoryModuleGetter{
+ dir: dir,
+ modulePath: modulePath,
+ }, nil
+}
+
+func (g *directoryModuleGetter) checkPath(path string) error {
+ if path != g.modulePath {
+ return fmt.Errorf("given module path %q does not match %q for directory %q: %w",
+ path, g.modulePath, g.dir, derrors.NotFound)
+ }
+ return nil
+}
+
+// Info returns basic information about the module.
+func (g *directoryModuleGetter) Info(ctx context.Context, path, version string) (*proxy.VersionInfo, error) {
+ if err := g.checkPath(path); err != nil {
+ return nil, err
+ }
+ return &proxy.VersionInfo{
+ Version: LocalVersion,
+ Time: LocalCommitTime,
+ }, nil
+}
+
+// Mod returns the contents of the module's go.mod file.
+// If the file does not exist, it returns a synthesized one.
+func (g *directoryModuleGetter) Mod(ctx context.Context, path, version string) ([]byte, error) {
+ if err := g.checkPath(path); err != nil {
+ return nil, err
+ }
+ data, err := ioutil.ReadFile(filepath.Join(g.dir, "go.mod"))
+ if errors.Is(err, os.ErrNotExist) {
+ return []byte(fmt.Sprintf("module %s\n", g.modulePath)), nil
+ }
+ return data, err
+}
+
+// FS returns an fs.FS for the module.
+func (g *directoryModuleGetter) FS(ctx context.Context, path, version string) (fs.FS, error) {
+ if err := g.checkPath(path); err != nil {
+ return nil, err
+ }
+ return createZipReader(g.dir, path, LocalVersion)
+}
+
+// ZipSize returns the approximate size of the zip file in bytes.
+func (g *directoryModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
+ return 0, errors.New("directoryModuleGetter.ZipSize unimplemented")
+}
+
+// createZipReader creates a zip file from a directory given a local path and
+// returns a zip.Reader to be passed to processZipFile. The purpose of the
+// function is to transform a local go module into a zip file to be processed by
+// existing functions.
+func createZipReader(localPath, modulePath, version string) (*zip.Reader, error) {
+ buf := new(bytes.Buffer)
+ w := zip.NewWriter(buf)
+ err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+
+ readFrom, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer readFrom.Close()
+
+ writeTo, err := w.Create(filepath.Join(moduleVersionDir(modulePath, version), strings.TrimPrefix(path, localPath)))
+ if err != nil {
+ return err
+ }
+
+ _, err = io.Copy(writeTo, readFrom)
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+ if err := w.Close(); err != nil {
+ return nil, err
+ }
+
+ reader := bytes.NewReader(buf.Bytes())
+ return zip.NewReader(reader, reader.Size())
+}
+
+// An fsModuleGetter gets modules from a directory in the filesystem
+// that is organized like the proxy, with paths that correspond to proxy
+// URLs. An example of such a directory is $(go env GOMODCACHE)/cache/download.
+type fsModuleGetter struct {
+ dir string
+}
+
+// NewFSModuleGetter return a ModuleGetter that reads modules from a filesystem
+// directory organized like the proxy.
+func NewFSModuleGetter(dir string) ModuleGetter {
+ return &fsModuleGetter{dir: dir}
+}
+
+// Info returns basic information about the module.
+func (g *fsModuleGetter) Info(ctx context.Context, path, version string) (_ *proxy.VersionInfo, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.Info(%q, %q)", path, version)
+
+ data, err := g.readFile(path, version, "info")
+ if err != nil {
+ return nil, err
+ }
+ var info proxy.VersionInfo
+ if err := json.Unmarshal(data, &info); err != nil {
+ return nil, err
+ }
+ return &info, nil
+}
+
+// Mod returns the contents of the module's go.mod file.
+func (g *fsModuleGetter) Mod(ctx context.Context, path, version string) (_ []byte, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.Mod(%q, %q)", path, version)
+
+ return g.readFile(path, version, "mod")
+}
+
+// FS returns an FS for the module's zip file.
+func (g *fsModuleGetter) FS(ctx context.Context, path, version string) (_ fs.FS, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.FS(%q, %q)", path, version)
+
+ data, err := g.readFile(path, version, "zip")
+ if err != nil {
+ return nil, err
+ }
+ return zip.NewReader(bytes.NewReader(data), int64(len(data)))
+}
+
+// ZipSize returns the approximate size of the zip file in bytes.
+func (g *fsModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
+ return 0, errors.New("fsModuleGetter.ZipSize unimplemented")
+}
+
+func (g *fsModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
+ epath, err := g.escapedPath(path, version, suffix)
+ if err != nil {
+ return nil, err
+ }
+ f, err := os.Open(epath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return ioutil.ReadAll(f)
+}
+
+func (g *fsModuleGetter) escapedPath(modulePath, version, suffix string) (string, error) {
+ ep, err := module.EscapePath(modulePath)
+ if err != nil {
+ return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
+ }
+ ev, err := module.EscapeVersion(version)
+ if err != nil {
+ return "", fmt.Errorf("version: %v: %w", err, derrors.InvalidArgument)
+ }
+ return filepath.Join(g.dir, ep, "@v", fmt.Sprintf("%s.%s", ev, suffix)), nil
+}
diff --git a/internal/fetch/fetchlocal_test.go b/internal/fetch/getters_test.go
similarity index 100%
rename from internal/fetch/fetchlocal_test.go
rename to internal/fetch/getters_test.go
