internal/fetch: add a ModuleGetter for a proxy-like filesystem
Add fsModuleGetter, which is a ModuleGetter for a directory organized
like proxy URLs. The go module download cache is in this format.
For golang/go#47780
Change-Id: Ifccdfa3010874adddc0a600593699d454e44c0e3
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/343220
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
TryBot-Result: kokoro <noreply+kokoro@google.com>
diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go
index b54dd6f..91806f7 100644
--- a/internal/fetch/fetch.go
+++ b/internal/fetch/fetch.go
@@ -94,7 +94,6 @@
// Info returns basic information about the module.
Info(ctx context.Context, path, version string) (*proxy.VersionInfo, error)
// Mod returns the contents of the module's go.mod file.
- // If the file does not exist, it returns a synthesized one.
Mod(ctx context.Context, path, version string) ([]byte, error)
// Zip returns a reader for the module's zip file.
Zip(ctx context.Context, path, version string) (*zip.Reader, error)
diff --git a/internal/fetch/fs.go b/internal/fetch/fs.go
new file mode 100644
index 0000000..7624119
--- /dev/null
+++ b/internal/fetch/fs.go
@@ -0,0 +1,97 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fetch
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/proxy"
+)
+
+// An fsModuleGetter gets modules from a directory in the filesystem
+// that is organized like the proxy, with paths that correspond to proxy
+// URLs. An example of such a directory is $(go env GOMODCACHE)/cache/download.
+type fsModuleGetter struct {
+ dir string
+}
+
+// NewFSModuleGetter return a ModuleGetter that reads modules from a filesystem
+// directory organized like the proxy.
+func NewFSModuleGetter(dir string) ModuleGetter {
+ return &fsModuleGetter{dir: dir}
+}
+
+// Info returns basic information about the module.
+func (g *fsModuleGetter) Info(ctx context.Context, path, version string) (_ *proxy.VersionInfo, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.Info(%q, %q)", path, version)
+
+ data, err := g.readFile(path, version, "info")
+ if err != nil {
+ return nil, err
+ }
+ var info proxy.VersionInfo
+ if err := json.Unmarshal(data, &info); err != nil {
+ return nil, err
+ }
+ return &info, nil
+}
+
+// Mod returns the contents of the module's go.mod file.
+func (g *fsModuleGetter) Mod(ctx context.Context, path, version string) (_ []byte, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.Mod(%q, %q)", path, version)
+
+ return g.readFile(path, version, "mod")
+}
+
+// Zip returns a reader for the module's zip file.
+func (g *fsModuleGetter) Zip(ctx context.Context, path, version string) (_ *zip.Reader, err error) {
+ defer derrors.Wrap(&err, "fsModuleGetter.Zip(%q, %q)", path, version)
+
+ data, err := g.readFile(path, version, "zip")
+ if err != nil {
+ return nil, err
+ }
+ return zip.NewReader(bytes.NewReader(data), int64(len(data)))
+}
+
+// ZipSize returns the approximate size of the zip file in bytes.
+func (g *fsModuleGetter) ZipSize(ctx context.Context, path, version string) (int64, error) {
+ return 0, errors.New("fsModuleGetter.ZipSize unimplemented")
+}
+
+func (g *fsModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
+ epath, err := g.escapedPath(path, version, suffix)
+ if err != nil {
+ return nil, err
+ }
+ f, err := os.Open(epath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return ioutil.ReadAll(f)
+}
+
+func (g *fsModuleGetter) escapedPath(modulePath, version, suffix string) (string, error) {
+ ep, err := module.EscapePath(modulePath)
+ if err != nil {
+ return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
+ }
+ ev, err := module.EscapeVersion(version)
+ if err != nil {
+ return "", fmt.Errorf("version: %v: %w", err, derrors.InvalidArgument)
+ }
+ return filepath.Join(g.dir, ep, "@v", fmt.Sprintf("%s.%s", ev, suffix)), nil
+}
diff --git a/internal/fetch/fs_test.go b/internal/fetch/fs_test.go
new file mode 100644
index 0000000..c4e3988
--- /dev/null
+++ b/internal/fetch/fs_test.go
@@ -0,0 +1,102 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fetch
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite/internal/proxy"
+)
+
+func TestEscapedPath(t *testing.T) {
+ for _, test := range []struct {
+ path, version, suffix string
+ want string
+ }{
+ {
+ "m.com", "v1", "info",
+ "dir/m.com/@v/v1.info",
+ },
+ {
+ "github.com/aBc", "v2.3.4", "zip",
+ "dir/github.com/a!bc/@v/v2.3.4.zip",
+ },
+ } {
+ g := NewFSModuleGetter("dir").(*fsModuleGetter)
+ got, err := g.escapedPath(test.path, test.version, test.suffix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != test.want {
+ t.Errorf("%s, %s, %s: got %q, want %q", test.path, test.version, test.suffix, got, test.want)
+ }
+ }
+}
+
+func TestFSGetter(t *testing.T) {
+ ctx := context.Background()
+ const (
+ modulePath = "github.com/jackc/pgio"
+ version = "v1.0.0"
+ goMod = "module github.com/jackc/pgio\n\ngo 1.12\n"
+ )
+ ts, err := time.Parse(time.RFC3339, "2019-03-30T17:04:38Z")
+ if err != nil {
+ t.Fatal(err)
+ }
+ g := NewFSModuleGetter("testdata/modcache")
+ t.Run("info", func(t *testing.T) {
+ got, err := g.Info(ctx, modulePath, version)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := &proxy.VersionInfo{Version: version, Time: ts}
+ if !cmp.Equal(got, want) {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+ })
+ t.Run("mod", func(t *testing.T) {
+ got, err := g.Mod(ctx, modulePath, version)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []byte(goMod)
+ if !cmp.Equal(got, want) {
+ t.Errorf("got %q, want %q", got, want)
+ }
+ })
+ t.Run("zip", func(t *testing.T) {
+ zr, err := g.Zip(ctx, modulePath, version)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Just check that the go.mod file is there and has the right contents.
+ goModPath := fmt.Sprintf("%s@%s/go.mod", modulePath, version)
+ for _, f := range zr.File {
+ if f.Name == goModPath {
+ f, err := f.Open()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ got, err := ioutil.ReadAll(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []byte(goMod)
+ if !cmp.Equal(got, want) {
+ t.Errorf("got %q, want %q", got, want)
+ }
+ return
+ }
+ }
+ t.Fatal("go.mod not found")
+ })
+}
diff --git a/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.info b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.info
new file mode 100644
index 0000000..2e4d188
--- /dev/null
+++ b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.info
@@ -0,0 +1 @@
+{"Version":"v1.0.0","Time":"2019-03-30T17:04:38Z"}
\ No newline at end of file
diff --git a/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.mod b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.mod
new file mode 100644
index 0000000..c1efddd
--- /dev/null
+++ b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.mod
@@ -0,0 +1,3 @@
+module github.com/jackc/pgio
+
+go 1.12
diff --git a/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.zip b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.zip
new file mode 100644
index 0000000..9434545
--- /dev/null
+++ b/internal/fetch/testdata/modcache/github.com/jackc/pgio/@v/v1.0.0.zip
Binary files differ
