internal/modules: add package
Package modules is added to assist with fetching modules at version via
the proxy and writing them to disk.
Change-Id: I90f734c0f38af8925dd3f6bd6a03e586f49c03bf
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/464638
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Auto-Submit: Julie Qiu <julieqiu@google.com>
Run-TryBot: Julie Qiu <julieqiu@google.com>
Reviewed-by: Julie Qiu <julieqiu@google.com>
Run-TryBot: Julie Qiu <julie@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/internal/derrors/derrors.go b/internal/derrors/derrors.go
index 9dcb42c..f9514b7 100644
--- a/internal/derrors/derrors.go
+++ b/internal/derrors/derrors.go
@@ -35,6 +35,10 @@
// ProxyError is used to capture non-actionable server errors returned from the proxy.
ProxyError = errors.New("proxy error")
+
+ // ScanModuleOSError is used to capture issues with writing the module zip
+ // to disk during the scan setup process. This is not an error with vulncheck.
+ ScanModuleOSError = errors.New("scan module OS error")
)
// Wrap adds context to the error and allows
diff --git a/internal/modules/modules.go b/internal/modules/modules.go
new file mode 100644
index 0000000..5bc51ab
--- /dev/null
+++ b/internal/modules/modules.go
@@ -0,0 +1,75 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package modules assists in working with modules.
+package modules
+
+import (
+ "archive/zip"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+)
+
+// Download fetches module at version via proxyClient and writes the modules
+// down to disk at dir.
+func Download(ctx context.Context, module, version, dir string, proxyClient *proxy.Client, stripModulePrefix bool) error {
+ zipr, err := proxyClient.Zip(ctx, module, version)
+ if err != nil {
+ return fmt.Errorf("%v: %w", err, derrors.ProxyError)
+ }
+ log.Debugf(ctx, "writing module zip: %s@%s", module, version)
+ stripPrefix := ""
+ if stripModulePrefix {
+ stripPrefix = module + "@" + version + "/"
+ }
+ if err := writeZip(zipr, dir, stripPrefix); err != nil {
+ return fmt.Errorf("%v: %w", err, derrors.ScanModuleOSError)
+ }
+ return nil
+}
+
+func writeZip(r *zip.Reader, destination, stripPrefix string) error {
+ for _, f := range r.File {
+ name := strings.TrimPrefix(f.Name, stripPrefix)
+ fpath := filepath.Join(destination, name)
+ if !strings.HasPrefix(fpath, filepath.Clean(destination)+string(os.PathSeparator)) {
+ return fmt.Errorf("%s is an illegal filepath", fpath)
+ }
+ if f.FileInfo().IsDir() {
+ if err := os.MkdirAll(fpath, os.ModePerm); err != nil {
+ return err
+ }
+ continue
+ }
+ if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+ return err
+ }
+ outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+ if err != nil {
+ return err
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return err
+ }
+ if _, err := io.Copy(outFile, rc); err != nil {
+ return err
+ }
+ if err := outFile.Close(); err != nil {
+ return err
+ }
+ if err := rc.Close(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/modules/modules_test.go b/internal/modules/modules_test.go
new file mode 100644
index 0000000..2e9ccb7
--- /dev/null
+++ b/internal/modules/modules_test.go
@@ -0,0 +1,40 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package modules
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+)
+
+func TestDownload(t *testing.T) {
+ t.Skip()
+
+ tempDir, err := os.MkdirTemp("", "testModuleDownload")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ defer func() {
+ if err := os.RemoveAll(tempDir); err != nil {
+ t.Fatal(err)
+ }
+ }()
+
+ proxyClient, err := proxy.New("https://proxy.golang.org")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Use golang.org/x/net@v0.0.0-20221012135044-0b7e1fb9d458
+ module := "golang.org/x/net"
+ version := "v0.0.0-20221012135044-0b7e1fb9d458"
+ if err := Download(context.Background(), module, version, tempDir, proxyClient, true); err != nil {
+ t.Errorf("failed to download %v@%v: %v", module, version, err)
+ }
+}
