internal/proxy: add package
Package proxy provides a client for fetching data from the Go module
proxy.
This is copied from x/pkgsite with modifications to remove the
dependency on the x/pkgsite/internal/testing/sample package.
Change-Id: I7fa17331f70e746aa2020fe27837d14af1483689
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/463621
Reviewed-by: Julie Qiu <julieqiu@google.com>
Run-TryBot: Julie Qiu <julieqiu@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/go.mod b/go.mod
index 1856172..2183769 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,11 @@
require (
cloud.google.com/go/errorreporting v0.3.0
github.com/client9/misspell v0.3.4
+ github.com/google/go-cmp v0.5.9
+ go.opencensus.io v0.23.0
golang.org/x/mod v0.7.0
+ golang.org/x/net v0.5.0
+ golang.org/x/tools v0.5.0
honnef.co/go/tools v0.3.3
mvdan.cc/unparam v0.0.0-20230125043941-70a0ce6e7b95
)
@@ -16,17 +20,13 @@
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
- github.com/google/go-cmp v0.5.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
- go.opencensus.io v0.23.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect
- golang.org/x/net v0.5.0 // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
- golang.org/x/tools v0.5.0 // indirect
google.golang.org/api v0.102.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
diff --git a/internal/derrors/derrors.go b/internal/derrors/derrors.go
index bc29af1..9dcb42c 100644
--- a/internal/derrors/derrors.go
+++ b/internal/derrors/derrors.go
@@ -14,6 +14,29 @@
"cloud.google.com/go/errorreporting"
)
+var (
+ // NotFound indicates that a requested entity was not found (HTTP 404).
+ NotFound = errors.New("not found")
+
+ // NotFetched means that the proxy returned "not found" with the
+ // Disable-Module-Fetch header set. We don't know if the module really
+ // doesn't exist, or the proxy just didn't fetch it.
+ NotFetched = errors.New("not fetched by proxy")
+
+ // InvalidArgument indicates that the input into the request is invalid in
+ // some way (HTTP 400).
+ InvalidArgument = errors.New("invalid argument")
+
+ // BadModule indicates a problem with a module.
+ BadModule = errors.New("bad module")
+
+ // ProxyTimedOut indicates that a request timed out when fetching from the Module Mirror.
+ ProxyTimedOut = errors.New("proxy timed out")
+
+ // ProxyError is used to capture non-actionable server errors returned from the proxy.
+ ProxyError = errors.New("proxy error")
+)
+
// Wrap adds context to the error and allows
// unwrapping the result to recover the original error.
//
diff --git a/internal/proxy/cache.go b/internal/proxy/cache.go
new file mode 100644
index 0000000..b7da79d
--- /dev/null
+++ b/internal/proxy/cache.go
@@ -0,0 +1,92 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+ "archive/zip"
+ "sync"
+)
+
+type modver struct {
+ Path string
+ Version string
+}
+
+// cache caches proxy info, mod and zip calls.
+type cache struct {
+ mu sync.Mutex
+
+ infoCache map[modver]*VersionInfo
+ modCache map[modver][]byte
+
+ // One-element zip cache, to avoid a double download.
+ // See TestFetchAndUpdateStateCacheZip in internal/worker/fetch_test.go.
+ zipKey modver
+ zipReader *zip.Reader
+}
+
+func (c *cache) getInfo(modulePath, version string) *VersionInfo {
+ if c == nil {
+ return nil
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.infoCache[modver{Path: modulePath, Version: version}]
+}
+
+func (c *cache) putInfo(modulePath, version string, v *VersionInfo) {
+ if c == nil {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.infoCache == nil {
+ c.infoCache = map[modver]*VersionInfo{}
+ }
+ c.infoCache[modver{Path: modulePath, Version: version}] = v
+}
+
+func (c *cache) getMod(modulePath, version string) []byte {
+ if c == nil {
+ return nil
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.modCache[modver{Path: modulePath, Version: version}]
+}
+
+func (c *cache) putMod(modulePath, version string, b []byte) {
+ if c == nil {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.modCache == nil {
+ c.modCache = map[modver][]byte{}
+ }
+ c.modCache[modver{Path: modulePath, Version: version}] = b
+}
+
+func (c *cache) getZip(modulePath, version string) *zip.Reader {
+ if c == nil {
+ return nil
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.zipKey == (modver{Path: modulePath, Version: version}) {
+ return c.zipReader
+ }
+ return nil
+}
+
+func (c *cache) putZip(modulePath, version string, r *zip.Reader) {
+ if c == nil {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.zipKey = modver{Path: modulePath, Version: version}
+ c.zipReader = r
+}
diff --git a/internal/proxy/client.go b/internal/proxy/client.go
new file mode 100644
index 0000000..b32243c
--- /dev/null
+++ b/internal/proxy/client.go
@@ -0,0 +1,304 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package proxy provides a client for interacting with a proxy.
+package proxy
+
+import (
+ "archive/zip"
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "go.opencensus.io/plugin/ochttp"
+ "golang.org/x/mod/module"
+ "golang.org/x/net/context/ctxhttp"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/version"
+)
+
+// A Client is used by the fetch service to communicate with a module
+// proxy. It handles all methods defined by go help goproxy.
+type Client struct {
+ // URL of the module proxy web server
+ url string
+
+ // Client used for HTTP requests. It is mutable for testing purposes.
+ HTTPClient *http.Client
+
+ // Whether fetch should be disabled.
+ disableFetch bool
+
+ cache *cache
+}
+
+// A VersionInfo contains metadata about a given version of a module.
+type VersionInfo struct {
+ Version string
+ Time time.Time
+}
+
+// Setting this header to true prevents the proxy from fetching uncached
+// modules.
+const DisableFetchHeader = "Disable-Module-Fetch"
+
+// New constructs a *Client using the provided url, which is expected to
+// be an absolute URI that can be directly passed to http.Get.
+func New(u string) (_ *Client, err error) {
+ defer derrors.WrapStack(&err, "proxy.New(%q)", u)
+ return &Client{
+ url: strings.TrimRight(u, "/"),
+ HTTPClient: &http.Client{Transport: &ochttp.Transport{}},
+ disableFetch: false,
+ }, nil
+}
+
+// WithFetchDisabled returns a new client that sets the Disable-Module-Fetch
+// header so that the proxy does not fetch a module it doesn't already know
+// about.
+func (c *Client) WithFetchDisabled() *Client {
+ c2 := *c
+ c2.disableFetch = true
+ return &c2
+}
+
+// FetchDisabled reports whether proxy fetch is disabled.
+func (c *Client) FetchDisabled() bool {
+ return c.disableFetch
+}
+
+// WithCache returns a new client that caches some RPCs.
+func (c *Client) WithCache() *Client {
+ c2 := *c
+ c2.cache = &cache{}
+ return &c2
+}
+
+// Info makes a request to $GOPROXY/<module>/@v/<requestedVersion>.info and
+// transforms that data into a *VersionInfo.
+// If requestedVersion is internal.LatestVersion, it uses the proxy's @latest
+// endpoint instead.
+func (c *Client) Info(ctx context.Context, modulePath, requestedVersion string) (_ *VersionInfo, err error) {
+ defer func() {
+ // Don't report NotFetched, because it is the normal result of fetching
+ // an uncached module when fetch is disabled.
+ // Don't report timeouts, because they are relatively frequent and not actionable.
+ wrap := derrors.Wrap
+ if !errors.Is(err, derrors.NotFetched) && !errors.Is(err, derrors.ProxyTimedOut) && !errors.Is(err, derrors.NotFound) {
+ wrap = derrors.WrapAndReport
+ }
+ wrap(&err, "proxy.Client.Info(%q, %q)", modulePath, requestedVersion)
+ }()
+
+ if v := c.cache.getInfo(modulePath, requestedVersion); v != nil {
+ return v, nil
+ }
+ data, err := c.readBody(ctx, modulePath, requestedVersion, "info")
+ if err != nil {
+ return nil, err
+ }
+ var v VersionInfo
+ if err := json.Unmarshal(data, &v); err != nil {
+ return nil, err
+ }
+ c.cache.putInfo(modulePath, requestedVersion, &v)
+ return &v, nil
+}
+
+// Mod makes a request to $GOPROXY/<module>/@v/<resolvedVersion>.mod and returns the raw data.
+func (c *Client) Mod(ctx context.Context, modulePath, resolvedVersion string) (_ []byte, err error) {
+ defer derrors.WrapStack(&err, "proxy.Client.Mod(%q, %q)", modulePath, resolvedVersion)
+
+ if b := c.cache.getMod(modulePath, resolvedVersion); b != nil {
+ return b, nil
+ }
+ b, err := c.readBody(ctx, modulePath, resolvedVersion, "mod")
+ if err != nil {
+ return nil, err
+ }
+ c.cache.putMod(modulePath, resolvedVersion, b)
+ return b, nil
+}
+
+// Zip makes a request to $GOPROXY/<modulePath>/@v/<resolvedVersion>.zip and
+// transforms that data into a *zip.Reader. <resolvedVersion> must have already
+// been resolved by first making a request to
+// $GOPROXY/<modulePath>/@v/<requestedVersion>.info to obtained the valid
+// semantic version.
+func (c *Client) Zip(ctx context.Context, modulePath, resolvedVersion string) (_ *zip.Reader, err error) {
+ defer derrors.WrapStack(&err, "proxy.Client.Zip(ctx, %q, %q)", modulePath, resolvedVersion)
+
+ if r := c.cache.getZip(modulePath, resolvedVersion); r != nil {
+ return r, nil
+ }
+ bodyBytes, err := c.readBody(ctx, modulePath, resolvedVersion, "zip")
+ if err != nil {
+ return nil, err
+ }
+ zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), int64(len(bodyBytes)))
+ if err != nil {
+ return nil, fmt.Errorf("zip.NewReader: %v: %w", err, derrors.BadModule)
+ }
+ c.cache.putZip(modulePath, resolvedVersion, zipReader)
+ return zipReader, nil
+}
+
+// ZipSize gets the size in bytes of the zip from the proxy, without downloading it.
+// The version must be resolved, as by a call to Client.Info.
+func (c *Client) ZipSize(ctx context.Context, modulePath, resolvedVersion string) (_ int64, err error) {
+ defer derrors.WrapStack(&err, "proxy.Client.ZipSize(ctx, %q, %q)", modulePath, resolvedVersion)
+
+ url, err := c.EscapedURL(modulePath, resolvedVersion, "zip")
+ if err != nil {
+ return 0, err
+ }
+ res, err := ctxhttp.Head(ctx, c.HTTPClient, url)
+ if err != nil {
+ return 0, fmt.Errorf("ctxhttp.Head(ctx, client, %q): %v", url, err)
+ }
+ defer res.Body.Close()
+ if err := responseError(res, false); err != nil {
+ return 0, err
+ }
+ if res.ContentLength < 0 {
+ return 0, errors.New("unknown content length")
+ }
+ return res.ContentLength, nil
+}
+
+func (c *Client) EscapedURL(modulePath, requestedVersion, suffix string) (_ string, err error) {
+ defer derrors.WrapStack(&err, "Client.escapedURL(%q, %q, %q)", modulePath, requestedVersion, suffix)
+
+ if suffix != "info" && suffix != "mod" && suffix != "zip" {
+ return "", errors.New(`suffix must be "info", "mod" or "zip"`)
+ }
+ escapedPath, err := module.EscapePath(modulePath)
+ if err != nil {
+ return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
+ }
+ if requestedVersion == version.Latest {
+ if suffix != "info" {
+ return "", fmt.Errorf("cannot ask for latest with suffix %q", suffix)
+ }
+ return fmt.Sprintf("%s/%s/@latest", c.url, escapedPath), nil
+ }
+ escapedVersion, err := module.EscapeVersion(requestedVersion)
+ if err != nil {
+ return "", fmt.Errorf("version: %v: %w", err, derrors.InvalidArgument)
+ }
+ return fmt.Sprintf("%s/%s/@v/%s.%s", c.url, escapedPath, escapedVersion, suffix), nil
+}
+
+func (c *Client) readBody(ctx context.Context, modulePath, requestedVersion, suffix string) (_ []byte, err error) {
+ defer derrors.WrapStack(&err, "Client.readBody(%q, %q, %q)", modulePath, requestedVersion, suffix)
+
+ u, err := c.EscapedURL(modulePath, requestedVersion, suffix)
+ if err != nil {
+ return nil, err
+ }
+ var data []byte
+ err = c.executeRequest(ctx, u, func(body io.Reader) error {
+ var err error
+ data, err = io.ReadAll(body)
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+ return data, nil
+}
+
+// Versions makes a request to $GOPROXY/<path>/@v/list and returns the
+// resulting version strings.
+func (c *Client) Versions(ctx context.Context, modulePath string) (_ []string, err error) {
+ defer derrors.Wrap(&err, "Versions(ctx, %q)", modulePath)
+ escapedPath, err := module.EscapePath(modulePath)
+ if err != nil {
+ return nil, fmt.Errorf("module.EscapePath(%q): %w", modulePath, derrors.InvalidArgument)
+ }
+ u := fmt.Sprintf("%s/%s/@v/list", c.url, escapedPath)
+ var versions []string
+ collect := func(body io.Reader) error {
+ scanner := bufio.NewScanner(body)
+ for scanner.Scan() {
+ versions = append(versions, scanner.Text())
+ }
+ return scanner.Err()
+ }
+ if err := c.executeRequest(ctx, u, collect); err != nil {
+ return nil, err
+ }
+ return versions, nil
+}
+
+// executeRequest executes an HTTP GET request for u, then calls the bodyFunc
+// on the response body, if no error occurred.
+func (c *Client) executeRequest(ctx context.Context, u string, bodyFunc func(body io.Reader) error) (err error) {
+ defer func() {
+ if ctx.Err() != nil {
+ err = fmt.Errorf("%v: %w", err, derrors.ProxyTimedOut)
+ }
+ derrors.WrapStack(&err, "executeRequest(ctx, %q)", u)
+ }()
+
+ req, err := http.NewRequest("GET", u, nil)
+ if err != nil {
+ return err
+ }
+ if c.disableFetch {
+ req.Header.Set(DisableFetchHeader, "true")
+ }
+ r, err := ctxhttp.Do(ctx, c.HTTPClient, req)
+ if err != nil {
+ return fmt.Errorf("ctxhttp.Do(ctx, client, %q): %v", u, err)
+ }
+ defer r.Body.Close()
+ if err := responseError(r, c.disableFetch); err != nil {
+ return err
+ }
+ return bodyFunc(r.Body)
+}
+
+// responseError translates the response status code to an appropriate error.
+func responseError(r *http.Response, fetchDisabled bool) error {
+ switch {
+ case 200 <= r.StatusCode && r.StatusCode < 300:
+ return nil
+ case 500 <= r.StatusCode:
+ return derrors.ProxyError
+ case r.StatusCode == http.StatusNotFound,
+ r.StatusCode == http.StatusGone:
+ // Treat both 404 Not Found and 410 Gone responses
+ // from the proxy as a "not found" error category.
+ // If the response body contains "fetch timed out", treat this
+ // as a 504 response so that we retry fetching the module version again
+ // later.
+ //
+ // If the Disable-Module-Fetch header was set, use a different
+ // error code so we can tell the difference.
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ return fmt.Errorf("io.ReadAll: %v", err)
+ }
+ d := string(data)
+ switch {
+ case strings.Contains(d, "fetch timed out"):
+ err = derrors.ProxyTimedOut
+ case fetchDisabled:
+ err = derrors.NotFetched
+ default:
+ err = derrors.NotFound
+ }
+ return fmt.Errorf("%q: %w", d, err)
+ default:
+ return fmt.Errorf("unexpected status %d %s", r.StatusCode, r.Status)
+ }
+}
diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go
new file mode 100644
index 0000000..ad06ce0
--- /dev/null
+++ b/internal/proxy/client_test.go
@@ -0,0 +1,388 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy_test
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+ "golang.org/x/pkgsite-metrics/internal/proxy/proxytest"
+ "golang.org/x/pkgsite-metrics/internal/testing/testhelper"
+ "golang.org/x/pkgsite-metrics/internal/version"
+)
+
+const (
+ testTimeout = 5 * time.Second
+ testModulePath = "golang.org/x/module"
+ testVersion = "v1.0.0"
+)
+
+var testModule = &proxytest.Module{
+ ModulePath: testModulePath,
+ Version: testVersion,
+ Files: map[string]string{
+ "go.mod": "module github.com/my/module\n\ngo 1.12",
+ "LICENSE": testhelper.BSD0License,
+ "README.md": "README FILE FOR TESTING.",
+ "bar/LICENSE": testhelper.MITLicense,
+ "bar/bar.go": `
+ // package bar
+ package bar
+
+ // Bar returns the string "bar".
+ func Bar() string {
+ return "bar"
+ }`,
+ "foo/LICENSE.md": testhelper.MITLicense,
+ "foo/foo.go": `
+ // package foo
+ package foo
+
+ import (
+ "fmt"
+
+ "github.com/my/module/bar"
+ )
+
+ // FooBar returns the string "foo bar".
+ func FooBar() string {
+ return fmt.Sprintf("foo %s", bar.Bar())
+ }`,
+ },
+}
+
+const uncachedModulePath = "example.com/uncached"
+
+var uncachedModule = &proxytest.Module{
+ ModulePath: uncachedModulePath,
+ Version: testVersion,
+ NotCached: true,
+}
+
+func TestGetLatestInfo(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ testModules := []*proxytest.Module{
+ {
+ ModulePath: testModulePath,
+ Version: "v1.1.0",
+ Files: map[string]string{"bar.go": "package bar\nconst Version = 1.1"},
+ },
+ {
+ ModulePath: testModulePath,
+ Version: "v1.2.0",
+ Files: map[string]string{"bar.go": "package bar\nconst Version = 1.2"},
+ },
+ }
+ client, teardownProxy := proxytest.SetupTestClient(t, testModules)
+ defer teardownProxy()
+
+ info, err := client.Info(ctx, testModulePath, version.Latest)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if got, want := info.Version, "v1.2.0"; got != want {
+ t.Errorf("Info(ctx, %q, %q): Version = %q, want %q", testModulePath, version.Latest, got, want)
+ }
+}
+
+func TestListVersions(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ testModules := []*proxytest.Module{
+ {
+ ModulePath: testModulePath,
+ Version: "v1.1.0",
+ Files: map[string]string{"bar.go": "package bar\nconst Version = 1.1"},
+ },
+ {
+ ModulePath: testModulePath,
+ Version: "v1.2.0",
+ Files: map[string]string{"bar.go": "package bar\nconst Version = 1.2"},
+ },
+ {
+ ModulePath: testModulePath + "/bar",
+ Version: "v1.3.0",
+ Files: map[string]string{"bar.go": "package bar\nconst Version = 1.3"},
+ },
+ }
+ client, teardownProxy := proxytest.SetupTestClient(t, testModules)
+ defer teardownProxy()
+
+ want := []string{"v1.1.0", "v1.2.0"}
+ got, err := client.Versions(ctx, testModulePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("Versions(%q) diff:\n%s", testModulePath, diff)
+ }
+}
+
+func TestInfo(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ client, teardownProxy := proxytest.SetupTestClient(t, []*proxytest.Module{testModule, uncachedModule})
+ defer teardownProxy()
+
+ info, err := client.Info(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if info.Version != testVersion {
+ t.Errorf("VersionInfo.Version for Info(ctx, %q, %q) = %q, want %q",
+ testModulePath, testVersion, info.Version, testVersion)
+ }
+ expectedTime := time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC)
+ if info.Time != expectedTime {
+ t.Errorf("VersionInfo.Time for Info(ctx, %q, %q) = %v, want %v", testModulePath, testVersion, info.Time, expectedTime)
+ }
+
+ // With fetch disabled, Info returns "NotFetched" error on uncached module.
+ noFetchClient := client.WithFetchDisabled()
+ _, err = noFetchClient.Info(ctx, uncachedModulePath, testVersion)
+ if !errors.Is(err, derrors.NotFetched) {
+ t.Fatalf("got %v, want NotFetched", err)
+ }
+ // Info with fetch disabled succeeds on a cached module.
+ _, err = noFetchClient.Info(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestInfo_Errors(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ proxyServer := proxytest.NewServer(nil)
+ proxyServer.AddRoute(
+ fmt.Sprintf("/%s/@v/%s.info", "module.com/timeout", testVersion),
+ func(w http.ResponseWriter, r *http.Request) { http.Error(w, "fetch timed out", http.StatusNotFound) })
+ client, teardownProxy, err := proxytest.NewClientForServer(proxyServer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer teardownProxy()
+
+ for _, test := range []struct {
+ modulePath string
+ want error
+ }{
+ {
+ modulePath: testModulePath,
+ want: derrors.NotFound,
+ },
+ {
+ modulePath: "module.com/timeout",
+ want: derrors.ProxyTimedOut,
+ },
+ } {
+ if _, err := client.Info(ctx, test.modulePath, testVersion); !errors.Is(err, test.want) {
+ t.Errorf("Info(ctx, %q, %q): %v, want %v", test.modulePath, testVersion, err, test.want)
+ }
+ }
+}
+
+func TestMod(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ client, teardownProxy := proxytest.SetupTestClient(t, []*proxytest.Module{testModule})
+ defer teardownProxy()
+
+ bytes, err := client.Mod(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := string(bytes)
+ want := "module github.com/my/module\n\ngo 1.12"
+ if got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+func TestGetZip(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ client, teardownProxy := proxytest.SetupTestClient(t, []*proxytest.Module{testModule})
+ defer teardownProxy()
+
+ zipReader, err := client.Zip(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ wantFiles := []string{
+ testModulePath + "@" + testVersion + "/LICENSE",
+ testModulePath + "@" + testVersion + "/README.md",
+ testModulePath + "@" + testVersion + "/go.mod",
+ testModulePath + "@" + testVersion + "/foo/foo.go",
+ testModulePath + "@" + testVersion + "/foo/LICENSE.md",
+ testModulePath + "@" + testVersion + "/bar/bar.go",
+ testModulePath + "@" + testVersion + "/bar/LICENSE",
+ }
+ if len(zipReader.File) != len(wantFiles) {
+ t.Errorf("Zip(ctx, %q, %q) returned number of files: got %d, want %d",
+ testModulePath, testVersion, len(zipReader.File), len(wantFiles))
+ }
+
+ expectedFileSet := map[string]bool{}
+ for _, ef := range wantFiles {
+ expectedFileSet[ef] = true
+ }
+ for _, zipFile := range zipReader.File {
+ if !expectedFileSet[zipFile.Name] {
+ t.Errorf("Zip(ctx, %q, %q) returned unexpected file: %q", testModulePath,
+ testVersion, zipFile.Name)
+ }
+ expectedFileSet[zipFile.Name] = false
+ }
+}
+
+func TestZipNonExist(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+ defer cancel()
+
+ client, teardownProxy := proxytest.SetupTestClient(t, nil)
+ defer teardownProxy()
+
+ if _, err := client.Zip(ctx, testModulePath, testVersion); !errors.Is(err, derrors.NotFound) {
+ t.Errorf("got %v, want %v", err, derrors.NotFound)
+ }
+}
+
+func TestZipSize(t *testing.T) {
+ // TODO: fix test
+ t.Skip()
+
+ t.Run("found", func(t *testing.T) {
+ client, teardownProxy := proxytest.SetupTestClient(t, []*proxytest.Module{testModule})
+ defer teardownProxy()
+ got, err := client.ZipSize(context.Background(), testModulePath, testVersion)
+ if err != nil {
+ t.Error(err)
+ }
+ const want = 3235
+ if got != want {
+ t.Errorf("got %d, want %d", got, want)
+ }
+ })
+ t.Run("not found", func(t *testing.T) {
+ client, teardownProxy := proxytest.SetupTestClient(t, nil)
+ defer teardownProxy()
+ if _, err := client.ZipSize(context.Background(), testModulePath, testVersion); !errors.Is(err, derrors.NotFound) {
+ t.Errorf("got %v, want %v", err, derrors.NotFound)
+ }
+ })
+}
+
+func TestEncodedURL(t *testing.T) {
+ c, err := proxy.New("u")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, test := range []struct {
+ path, version, suffix string
+ want string // empty => error
+ }{
+ {
+ "mod.com", "v1.0.0", "info",
+ "u/mod.com/@v/v1.0.0.info",
+ },
+ {
+ "mod", "v1.0.0", "info",
+ "", // bad module path
+ },
+ {
+ "mod.com", "v1.0.0-rc1", "info",
+ "u/mod.com/@v/v1.0.0-rc1.info",
+ },
+ {
+ "mod.com/Foo", "v1.0.0-RC1", "info",
+ "u/mod.com/!foo/@v/v1.0.0-!r!c1.info",
+ },
+ {
+ "mod.com", ".", "info",
+ "", // bad version
+ },
+ {
+ "mod.com", "v1.0.0", "zip",
+ "u/mod.com/@v/v1.0.0.zip",
+ },
+ {
+ "mod", "v1.0.0", "zip",
+ "", // bad module path
+ },
+ {
+ "mod.com", "v1.0.0-rc1", "zip",
+ "u/mod.com/@v/v1.0.0-rc1.zip",
+ },
+ {
+ "mod.com/Foo", "v1.0.0-RC1", "zip",
+ "u/mod.com/!foo/@v/v1.0.0-!r!c1.zip",
+ },
+ {
+ "mod.com", ".", "zip",
+ "", // bad version
+ },
+ {
+ "mod.com", version.Latest, "info",
+ "u/mod.com/@latest",
+ },
+ {
+ "mod.com", version.Latest, "zip",
+ "", // can't ask for latest zip
+ },
+ {
+ "mod.com", "v1.0.0", "other",
+ "", // only "info" or "zip"
+ },
+ } {
+ got, err := c.EscapedURL(test.path, test.version, test.suffix)
+ if got != test.want || (err != nil) != (test.want == "") {
+ t.Errorf("%s, %s, %s: got (%q, %v), want %q", test.path, test.version, test.suffix, got, err, test.want)
+ }
+ }
+}
+
+func TestCache(t *testing.T) {
+ ctx := context.Background()
+ c1, teardownProxy := proxytest.SetupTestClient(t, []*proxytest.Module{testModule})
+
+ c := c1.WithCache()
+ got, err := c.Info(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = got
+ teardownProxy()
+ // Need server to satisfy different request.
+ _, err = c.Info(ctx, testModulePath, "v4.5.6")
+ if err == nil {
+ t.Fatal("got nil, want error")
+ }
+ // Don't need server for cached request.
+ got2, err := c.Info(ctx, testModulePath, testVersion)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmp.Equal(got, got2) {
+ t.Errorf("got %+v first, then %+v", got, got2)
+ }
+}
diff --git a/internal/proxy/proxytest/module.go b/internal/proxy/proxytest/module.go
new file mode 100644
index 0000000..6df8ef0
--- /dev/null
+++ b/internal/proxy/proxytest/module.go
@@ -0,0 +1,72 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package proxytest supports testing with the proxy.
+package proxytest
+
+import "fmt"
+
+// Module represents a module version used by the proxy server.
+type Module struct {
+ ModulePath string
+ Version string
+ Files map[string]string
+ NotCached bool // if true, behaves like it's uncached
+ zip []byte
+}
+
+// ChangePath returns a copy of m with a different module path.
+func (m *Module) ChangePath(modulePath string) *Module {
+ m2 := *m
+ m2.ModulePath = modulePath
+ return &m2
+}
+
+// ChangeVersion returns a copy of m with a different version.
+func (m *Module) ChangeVersion(version string) *Module {
+ m2 := *m
+ m2.Version = version
+ return &m2
+}
+
+// AddFile returns a copy of m with an additional file. It
+// panics if the filename is already present.
+func (m *Module) AddFile(filename, contents string) *Module {
+ return m.setFile(filename, &contents, false)
+}
+
+// DeleteFile returns a copy of m with filename removed.
+// It panics if filename is not present.
+func (m *Module) DeleteFile(filename string) *Module {
+ return m.setFile(filename, nil, true)
+}
+
+// ReplaceFile returns a copy of m with different contents for filename.
+// It panics if filename is not present.
+func (m *Module) ReplaceFile(filename, contents string) *Module {
+ return m.setFile(filename, &contents, true)
+}
+
+func (m *Module) setFile(filename string, contents *string, mustExist bool) *Module {
+ _, ok := m.Files[filename]
+ if mustExist && !ok {
+ panic(fmt.Sprintf("%s@%s does not have a file named %s", m.ModulePath, m.Version, filename))
+ }
+ if !mustExist && ok {
+ panic(fmt.Sprintf("%s@%s already has a file named %s", m.ModulePath, m.Version, filename))
+ }
+ m2 := *m
+ if m.Files != nil {
+ m2.Files = map[string]string{}
+ for k, v := range m.Files {
+ m2.Files[k] = v
+ }
+ }
+ if contents == nil {
+ delete(m2.Files, filename)
+ } else {
+ m2.Files[filename] = *contents
+ }
+ return &m2
+}
diff --git a/internal/proxy/proxytest/proxytest.go b/internal/proxy/proxytest/proxytest.go
new file mode 100644
index 0000000..4dce309
--- /dev/null
+++ b/internal/proxy/proxytest/proxytest.go
@@ -0,0 +1,116 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxytest
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "golang.org/x/mod/modfile"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+ "golang.org/x/pkgsite-metrics/internal/testing/testhelper"
+ "golang.org/x/tools/txtar"
+)
+
+// SetupTestClient creates a fake module proxy for testing using the given test
+// version information.
+//
+// It returns a function for tearing down the proxy after the test is completed
+// and a Client for interacting with the test proxy.
+func SetupTestClient(t *testing.T, modules []*Module) (*proxy.Client, func()) {
+ t.Helper()
+ s := NewServer(modules)
+ client, serverClose, err := NewClientForServer(s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return client, serverClose
+}
+
+// NewClientForServer starts serving proxyMux locally. It returns a client to the
+// server and a function to shut down the server.
+func NewClientForServer(s *Server) (*proxy.Client, func(), error) {
+ // override client.httpClient to skip TLS verification
+ httpClient, prox, serverClose := testhelper.SetupTestClientAndServer(s.mux)
+ client, err := proxy.New(prox.URL)
+ if err != nil {
+ return nil, nil, err
+ }
+ client.HTTPClient = httpClient
+ return client, serverClose, nil
+}
+
+// LoadTestModules reads the modules in the given directory. Each file in that
+// directory with a .txtar extension should be named "path@version" and should
+// be in txtar format (golang.org/x/tools/txtar). The path part of the filename
+// will be preceded by "example.com/" and colons will be replaced by slashes to
+// form a full module path. The file contents are used verbatim except that some
+// variables beginning with "$" are substituted with predefined strings.
+//
+// LoadTestModules panics if there is an error reading any of the files.
+func LoadTestModules(dir string) []*Module {
+ files, err := filepath.Glob(filepath.Join(dir, "*.txtar"))
+ if err != nil {
+ panic(err)
+ }
+ var ms []*Module
+ for _, f := range files {
+ m, err := readTxtarModule(f)
+ if err != nil {
+ panic(err)
+ }
+ ms = append(ms, m)
+ }
+ return ms
+}
+
+var testModuleReplacer = strings.NewReplacer(
+ "$MITLicense", testhelper.MITLicense,
+ "$BSD0License", testhelper.BSD0License,
+)
+
+func readTxtarModule(filename string) (*Module, error) {
+ modver := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+ i := strings.IndexRune(modver, '@')
+ if i < 0 {
+ return nil, fmt.Errorf("%s: filename missing '@'", modver)
+ }
+ modulePath, version := "example.com/"+modver[:i], modver[i+1:]
+ modulePath = strings.ReplaceAll(modulePath, ":", "/")
+ if modulePath == "" || version == "" {
+ return nil, fmt.Errorf("%s: empty module path or version", filename)
+ }
+ m := &Module{
+ ModulePath: modulePath,
+ Version: version,
+ Files: map[string]string{},
+ }
+ ar, err := txtar.ParseFile(filename)
+ if err != nil {
+ return nil, err
+ }
+ for _, f := range ar.Files {
+ if f.Name == "go.mod" {
+ // Overwrite the pregenerated module path if one is specified in
+ // the go.mod file.
+ m.ModulePath = modfile.ModulePath(f.Data)
+ }
+ m.Files[f.Name] = strings.TrimSpace(testModuleReplacer.Replace(string(f.Data)))
+ }
+ return m, nil
+}
+
+// FindModule returns the module in mods with the given path and version, or nil
+// if there isn't one. An empty version argument matches any version.
+func FindModule(mods []*Module, path, version string) *Module {
+ for _, m := range mods {
+ if m.ModulePath == path && (version == "" || m.Version == version) {
+ return m
+ }
+ }
+ return nil
+}
diff --git a/internal/proxy/proxytest/server.go b/internal/proxy/proxytest/server.go
new file mode 100644
index 0000000..d974c2e
--- /dev/null
+++ b/internal/proxy/proxytest/server.go
@@ -0,0 +1,194 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxytest
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/mod/semver"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+ "golang.org/x/pkgsite-metrics/internal/testing/testhelper"
+ "golang.org/x/pkgsite-metrics/internal/version"
+)
+
+// Server represents a proxy server containing the specified modules.
+type Server struct {
+ mu sync.Mutex
+ modules map[string][]*Module
+ mux *http.ServeMux
+ zipRequests int // number of .zip endpoint requests, for testing
+}
+
+// NewServer returns a proxy Server that serves the provided modules.
+func NewServer(modules []*Module) *Server {
+ s := &Server{
+ mux: http.NewServeMux(),
+ modules: map[string][]*Module{},
+ }
+ for _, m := range modules {
+ s.AddModule(m)
+ }
+ return s
+}
+
+// handleInfo creates an info endpoint for the specified module version.
+func (s *Server) handleInfo(modulePath, resolvedVersion string, uncached bool) {
+ urlPath := fmt.Sprintf("/%s/@v/%s.info", modulePath, resolvedVersion)
+ s.mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) {
+ if uncached && r.Header.Get(proxy.DisableFetchHeader) == "true" {
+ http.Error(w, "not found: temporarily unavailable", http.StatusGone)
+ return
+ }
+ http.ServeContent(w, r, modulePath, time.Now(), defaultInfo(resolvedVersion))
+ })
+}
+
+// handleLatest creates an info endpoint for the specified module at the latest
+// version.
+func (s *Server) handleLatest(modulePath, urlPath string) {
+ s.mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) {
+ modules := s.modules[modulePath]
+ resolvedVersion := modules[len(modules)-1].Version
+ http.ServeContent(w, r, modulePath, time.Now(), defaultInfo(resolvedVersion))
+ })
+}
+
+// handleMod creates a mod endpoint for the specified module version.
+func (s *Server) handleMod(m *Module) {
+ defaultGoMod := func(modulePath string) string {
+ // defaultGoMod creates a bare-bones go.mod contents.
+ return fmt.Sprintf("module %s\n\ngo 1.12", modulePath)
+ }
+ goMod := m.Files["go.mod"]
+ if goMod == "" {
+ goMod = defaultGoMod(m.ModulePath)
+ }
+ s.mux.HandleFunc(fmt.Sprintf("/%s/@v/%s.mod", m.ModulePath, m.Version),
+ func(w http.ResponseWriter, r *http.Request) {
+ http.ServeContent(w, r, m.ModulePath, time.Now(), strings.NewReader(goMod))
+ })
+}
+
+// handleZip creates a zip endpoint for the specified module version.
+func (s *Server) handleZip(m *Module) {
+ s.mux.HandleFunc(fmt.Sprintf("/%s/@v/%s.zip", m.ModulePath, m.Version),
+ func(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ s.zipRequests++
+ s.mu.Unlock()
+ http.ServeContent(w, r, m.ModulePath, time.Now(), bytes.NewReader(m.zip))
+ })
+}
+
+// handleList creates a list endpoint for the specified modulePath.
+func (s *Server) handleList(modulePath string) {
+ s.mux.HandleFunc(fmt.Sprintf("/%s/@v/list", modulePath), func(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ var vList []string
+ if modules, ok := s.modules[modulePath]; ok {
+ for _, v := range modules {
+ if !version.IsPseudo(v.Version) {
+ vList = append(vList, v.Version)
+ }
+ }
+ }
+ http.ServeContent(w, r, modulePath, time.Now(), strings.NewReader(strings.Join(vList, "\n")))
+ })
+}
+
+// AddRoute adds an additional handler to the server.
+func (s *Server) AddRoute(route string, fn func(w http.ResponseWriter, r *http.Request)) {
+ s.mux.HandleFunc(route, fn)
+}
+
+// AddModule adds an additional module to the server.
+func (s *Server) AddModule(m *Module) {
+ s.addModule(m, true)
+}
+
+// AddModuleNoLatest adds a module to the server, but the @v/list endpoint will
+// return nothing and @latest endpoint will serve a 410.
+// For testing the unusual case where a module exists but there is no version information.
+func (s *Server) AddModuleNoVersions(m *Module) {
+ s.addModule(m, false)
+}
+
+func (s *Server) addModule(m *Module, hasVersions bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ m = cleanModule(m)
+
+ if _, ok := s.modules[m.ModulePath]; !ok {
+ if hasVersions {
+ s.handleList(m.ModulePath)
+ s.handleLatest(m.ModulePath, fmt.Sprintf("/%s/@latest", m.ModulePath))
+ // TODO(https://golang.org/issue/39985): Add endpoint for handling
+ // master and main versions.
+ if m.Version != "master" {
+ s.handleLatest(m.ModulePath, fmt.Sprintf("/%s/@v/master.info", m.ModulePath))
+ }
+ if m.Version != "main" {
+ s.handleLatest(m.ModulePath, fmt.Sprintf("/%s/@v/main.info", m.ModulePath))
+ }
+ } else {
+ s.mux.HandleFunc(fmt.Sprintf("/%s/@v/list", m.ModulePath), func(w http.ResponseWriter, r *http.Request) {
+ http.ServeContent(w, r, m.ModulePath, time.Now(), strings.NewReader(""))
+ })
+ s.mux.HandleFunc(fmt.Sprintf("/%s/@latest", m.ModulePath), func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "not found", http.StatusGone)
+ })
+ }
+ }
+ s.handleInfo(m.ModulePath, m.Version, m.NotCached)
+ s.handleMod(m)
+ s.handleZip(m)
+
+ s.modules[m.ModulePath] = append(s.modules[m.ModulePath], m)
+ sort.Slice(s.modules[m.ModulePath], func(i, j int) bool {
+ // Return the modules in order of decreasing semver.
+ return semver.Compare(s.modules[m.ModulePath][i].Version, s.modules[m.ModulePath][j].Version) < 0
+ })
+}
+
+func (s *Server) ZipRequests() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.zipRequests
+}
+
+// CommitTime is the time returned by all calls to the .info endpoint.
+var CommitTime = time.Date(2019, time.January, 30, 0, 0, 0, 0, time.UTC)
+
+func cleanModule(m *Module) *Module {
+ if m.Version == "" {
+ m.Version = "v1.0.0"
+ }
+
+ files := map[string]string{}
+ for path, contents := range m.Files {
+ p := m.ModulePath + "@" + m.Version + "/" + path
+ files[p] = contents
+ }
+ zip, err := testhelper.ZipContents(files)
+ if err != nil {
+ panic(err)
+ }
+ m.zip = zip
+ return m
+}
+
+func defaultInfo(resolvedVersion string) *strings.Reader {
+ return strings.NewReader(fmt.Sprintf("{\n\t\"Version\": %q,\n\t\"Time\": %q\n}",
+ resolvedVersion, CommitTime.Format(time.RFC3339)))
+}
diff --git a/internal/proxy/testdata/basic@v1.0.0.txtar b/internal/proxy/testdata/basic@v1.0.0.txtar
new file mode 100644
index 0000000..9e3d803
--- /dev/null
+++ b/internal/proxy/testdata/basic@v1.0.0.txtar
@@ -0,0 +1,56 @@
+A simple module with a single package, which is at the module root.
+
+-- go.mod --
+module example.com/basic
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- file1.go --
+// Package basic is a sample package.
+package basic
+
+import "time"
+
+// Version is the same as the module version.
+const Version = "v1.0.0"
+
+// F is a function.
+func F(t time.Time, s string) (T, u) {
+ x := 3
+ x = C
+}
+
+// G is new in v1.1.0.
+func G() int {
+ return 3
+}
+
+-- file2.go --
+package basic
+
+var V = Version
+
+type T int
+
+type u int
+
+-- example_test.go --
+package basic_test
+
+// Example for the package.
+func Example() {
+ fmt.Println("hello")
+ // Output: hello
+}
+
+// A function example.
+func ExampleF() {
+ basic.F()
+}
+
+
+
diff --git a/internal/proxy/testdata/basic@v1.1.0.txtar b/internal/proxy/testdata/basic@v1.1.0.txtar
new file mode 100644
index 0000000..52d75d5
--- /dev/null
+++ b/internal/proxy/testdata/basic@v1.1.0.txtar
@@ -0,0 +1,56 @@
+A simple module with a single package, which is at the module root.
+
+-- go.mod --
+module example.com/basic
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- file1.go --
+// Package basic is a sample package.
+package basic
+
+import "time"
+
+// Version is the same as the module version.
+const Version = "v1.1.0"
+
+// F is a function.
+func F(t time.Time, s string) (T, u) {
+ x := 3
+ x = C
+}
+
+// G is new in v1.1.0.
+func G() int {
+ return 3
+}
+
+-- file2.go --
+package basic
+
+var V = Version
+
+type T int
+
+type u int
+
+-- example_test.go --
+package basic_test
+
+// Example for the package.
+func Example() {
+ fmt.Println("hello")
+ // Output: hello
+}
+
+// A function example.
+func ExampleF() {
+ basic.F()
+}
+
+
+
diff --git a/internal/proxy/testdata/build-constraints@v1.0.0.txtar b/internal/proxy/testdata/build-constraints@v1.0.0.txtar
new file mode 100644
index 0000000..f92ca74
--- /dev/null
+++ b/internal/proxy/testdata/build-constraints@v1.0.0.txtar
@@ -0,0 +1,34 @@
+A module with files that have build constraints.
+
+-- go.mod --
+module example.com/build-constraints
+
+-- LICENSE --
+$BSD0License
+
+-- cpu/cpu.go --
+// Package cpu implements processor feature detection
+// used by the Go standard library.
+package cpu
+
+-- cpu/cpu_arm.go --
+package cpu
+
+nconst CacheLinePadSize = 1
+
+-- cpu/cpu_arm64.go --
+package cpu
+
+const CacheLinePadSize = 2
+
+-- cpu/cpu_x86.go --
+// +build 386 amd64 amd64p32
+
+package cpu
+
+const CacheLinePadSize = 3
+
+-- ignore/ignore.go --
+// +build ignore
+
+package ignore
diff --git a/internal/proxy/testdata/deprecated@v1.0.0.txtar b/internal/proxy/testdata/deprecated@v1.0.0.txtar
new file mode 100644
index 0000000..6ff8146
--- /dev/null
+++ b/internal/proxy/testdata/deprecated@v1.0.0.txtar
@@ -0,0 +1,15 @@
+A module that is deprecated, according to its latest go.mod file
+(not this one).
+
+-- go.mod --
+module example.com/deprecated
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+// Package pkg is a sample package.
+package pkg
+
+// Version is the same as the module version.
+const Version = "v1.0.0"
diff --git a/internal/proxy/testdata/deprecated@v1.1.0.txtar b/internal/proxy/testdata/deprecated@v1.1.0.txtar
new file mode 100644
index 0000000..b88033f
--- /dev/null
+++ b/internal/proxy/testdata/deprecated@v1.1.0.txtar
@@ -0,0 +1,15 @@
+A module that is deprecated, according to its latest go.mod file.
+
+-- go.mod --
+// Deprecated: use something else
+module example.com/deprecated
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+// Package pkg is a sample package.
+package pkg
+
+// Version is the same as the module version.
+const Version = "v1.1.0"
diff --git a/internal/proxy/testdata/generics@v1.0.0.txtar b/internal/proxy/testdata/generics@v1.0.0.txtar
new file mode 100644
index 0000000..8301701
--- /dev/null
+++ b/internal/proxy/testdata/generics@v1.0.0.txtar
@@ -0,0 +1,29 @@
+A module that uses generics.
+
+-- go.mod --
+module example.com/generics
+
+go 1.18
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+
+// Package generics uses generics.
+package generics
+
+import "constraints"
+
+func Min[T constraints.Ordered](a, b T) T {
+ if a < b {
+ return a
+ }
+ return b
+}
+
+type List[T any] struct {
+ Val T
+ Next *List[T]
+}
+
diff --git a/internal/proxy/testdata/multi@v1.0.0.txtar b/internal/proxy/testdata/multi@v1.0.0.txtar
new file mode 100644
index 0000000..8512801
--- /dev/null
+++ b/internal/proxy/testdata/multi@v1.0.0.txtar
@@ -0,0 +1,45 @@
+A module with two packages, foo and bar.
+
+-- go.mod --
+module example.com/multi
+
+go 1.13
+
+-- LICENSE --
+$BSD0License
+
+-- README.md --
+README file for testing.
+
+-- foo/LICENSE.md --
+$MITLicense
+
+-- foo/foo.go --
+// package foo
+package foo
+
+import (
+ "fmt"
+
+ "example.com/multi/bar"
+)
+
+// FooBar returns the string "foo bar".
+func FooBar() string {
+ return fmt.Sprintf("foo %s", bar.Bar())
+}
+
+-- bar/LICENSE --
+$MITLicense
+
+-- bar/README --
+Another README file for testing.
+
+-- bar/bar.go --
+// package bar
+package bar
+
+// Bar returns the string "bar".
+func Bar() string {
+ return "bar"
+}
diff --git a/internal/proxy/testdata/nonredist@v1.0.0.txtar b/internal/proxy/testdata/nonredist@v1.0.0.txtar
new file mode 100644
index 0000000..ed58d6c
--- /dev/null
+++ b/internal/proxy/testdata/nonredist@v1.0.0.txtar
@@ -0,0 +1,57 @@
+A module with multiple packages, one of which is not redistributable.
+
+-- go.mod --
+module example.com/nonredist
+
+go 1.13
+
+-- LICENSE --
+$BSD0License
+
+-- README.md --
+README file for testing.
+
+-- bar/LICENSE --
+$MITLicense
+
+-- bar/bar.go --
+// package bar
+package bar
+
+// Bar returns the string "bar".
+func Bar() string {
+ return "bar"
+}
+
+
+-- bar/baz/COPYING --
+$MITLicense
+-- bar/baz/baz.go --
+// package baz
+package baz
+
+// Baz returns the string "baz".
+func Baz() string {
+ return "baz"
+}
+
+-- unk/README.md --
+README file will be removed before DB insert.
+
+-- unk/LICENSE.md --
+An unknown license.
+
+-- unk/unk.go --
+// package unk
+package unk
+
+import (
+ "fmt"
+
+ "example.com/nonredist/bar"
+)
+
+// FooBar returns the string "foo bar".
+func FooBar() string {
+ return fmt.Sprintf("foo %s", bar.Bar())
+}
diff --git a/internal/proxy/testdata/quote@v1.0.0.txtar b/internal/proxy/testdata/quote@v1.0.0.txtar
new file mode 100644
index 0000000..06276b2
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.0.0.txtar
@@ -0,0 +1,14 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello, world."
+}
diff --git a/internal/proxy/testdata/quote@v1.1.0.txtar b/internal/proxy/testdata/quote@v1.1.0.txtar
new file mode 100644
index 0000000..0b68637
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.1.0.txtar
@@ -0,0 +1,20 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello, world."
+}
+
+// Glass returns a useful phrase for world travelers.
+func Glass() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
diff --git a/internal/proxy/testdata/quote@v1.2.0.txtar b/internal/proxy/testdata/quote@v1.2.0.txtar
new file mode 100644
index 0000000..a736c15
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.2.0.txtar
@@ -0,0 +1,25 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello, world."
+}
+
+// Glass returns a useful phrase for world travelers.
+func Glass() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func Go() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
diff --git a/internal/proxy/testdata/quote@v1.3.0.txtar b/internal/proxy/testdata/quote@v1.3.0.txtar
new file mode 100644
index 0000000..912f527
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.3.0.txtar
@@ -0,0 +1,31 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello, world."
+}
+
+// Glass returns a useful phrase for world travelers.
+func Glass() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func Go() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
+
+// Opt returns an optimization truth.
+func Opt() string {
+ // Wisdom from ken.
+ return "If a program is too slow, it must have a loop."
+}
diff --git a/internal/proxy/testdata/quote@v1.4.0.txtar b/internal/proxy/testdata/quote@v1.4.0.txtar
new file mode 100644
index 0000000..e20f32d
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.4.0.txtar
@@ -0,0 +1,33 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+import "rsc.io/sampler"
+
+// Hello returns a greeting.
+func Hello() string {
+ return sampler.Hello()
+}
+
+// Glass returns a useful phrase for world travelers.
+func Glass() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func Go() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
+
+// Opt returns an optimization truth.
+func Opt() string {
+ // Wisdom from ken.
+ return "If a program is too slow, it must have a loop."
+}
diff --git a/internal/proxy/testdata/quote@v1.5.0.txtar b/internal/proxy/testdata/quote@v1.5.0.txtar
new file mode 100644
index 0000000..e20f32d
--- /dev/null
+++ b/internal/proxy/testdata/quote@v1.5.0.txtar
@@ -0,0 +1,33 @@
+-- go.mod --
+module rsc.io/quote
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+import "rsc.io/sampler"
+
+// Hello returns a greeting.
+func Hello() string {
+ return sampler.Hello()
+}
+
+// Glass returns a useful phrase for world travelers.
+func Glass() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func Go() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
+
+// Opt returns an optimization truth.
+func Opt() string {
+ // Wisdom from ken.
+ return "If a program is too slow, it must have a loop."
+}
diff --git a/internal/proxy/testdata/quote@v3.0.0.txtar b/internal/proxy/testdata/quote@v3.0.0.txtar
new file mode 100644
index 0000000..e3aaf9c
--- /dev/null
+++ b/internal/proxy/testdata/quote@v3.0.0.txtar
@@ -0,0 +1,33 @@
+-- go.mod --
+module rsc.io/quote/v3
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+import "rsc.io/sampler"
+
+// Hello returns a greeting.
+func HelloV3() string {
+ return sampler.Hello()
+}
+
+// Glass returns a useful phrase for world travelers.
+func GlassV3() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func GoV3() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
+
+// Opt returns an optimization truth.
+func OptV3() string {
+ // Wisdom from ken.
+ return "If a program is too slow, it must have a loop."
+}
diff --git a/internal/proxy/testdata/quote@v3.1.0.txtar b/internal/proxy/testdata/quote@v3.1.0.txtar
new file mode 100644
index 0000000..af33537
--- /dev/null
+++ b/internal/proxy/testdata/quote@v3.1.0.txtar
@@ -0,0 +1,38 @@
+-- go.mod --
+module rsc.io/quote/v3
+
+-- LICENSE --
+$MITLicense
+
+-- quote.go --
+// Package quote collects pithy sayings.
+package quote // import "rsc.io/quote"
+
+import "rsc.io/sampler"
+
+// Hello returns a greeting.
+func HelloV3() string {
+ return sampler.Hello()
+}
+
+// Concurrency returns a Go proverb about concurrency.
+func Concurrency() string {
+ return "Concurrency is not parallelism."
+}
+
+// Glass returns a useful phrase for world travelers.
+func GlassV3() string {
+ // See http://www.oocities.org/nodotus/hbglass.html.
+ return "I can eat glass and it doesn't hurt me."
+}
+
+// Go returns a Go proverb.
+func GoV3() string {
+ return "Don't communicate by sharing memory, share memory by communicating."
+}
+
+// Opt returns an optimization truth.
+func OptV3() string {
+ // Wisdom from ken.
+ return "If a program is too slow, it must have a loop."
+}
diff --git a/internal/proxy/testdata/retractions@v1.0.0.txtar b/internal/proxy/testdata/retractions@v1.0.0.txtar
new file mode 100644
index 0000000..8511567
--- /dev/null
+++ b/internal/proxy/testdata/retractions@v1.0.0.txtar
@@ -0,0 +1,17 @@
+A module with some versions retracted.
+This is not the latest version, so the retract directive is ignored.
+
+-- go.mod --
+module example.com/retractions
+
+retract v1.0.0
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+// Package pkg is a sample package.
+package pkg
+
+// Version is the same as the module version.
+const Version = "v1.0.0"
diff --git a/internal/proxy/testdata/retractions@v1.1.0.txtar b/internal/proxy/testdata/retractions@v1.1.0.txtar
new file mode 100644
index 0000000..97dc680
--- /dev/null
+++ b/internal/proxy/testdata/retractions@v1.1.0.txtar
@@ -0,0 +1,15 @@
+A module with some versions retracted.
+This is not the latest version.
+
+-- go.mod --
+module example.com/retractions
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+// Package pkg is a sample package.
+package pkg
+
+// Version is the same as the module version.
+const Version = "v1.1.0"
diff --git a/internal/proxy/testdata/retractions@v1.2.0.txtar b/internal/proxy/testdata/retractions@v1.2.0.txtar
new file mode 100644
index 0000000..4a19e67
--- /dev/null
+++ b/internal/proxy/testdata/retractions@v1.2.0.txtar
@@ -0,0 +1,20 @@
+A module with some versions retracted.
+This is the latest version. It retracts itself.
+
+-- go.mod --
+module example.com/retractions
+
+retract (
+ v1.2.0 // bad
+ v1.1.0 // worse
+)
+
+-- LICENSE --
+$MITLicense
+
+-- file.go --
+// Package pkg is a sample package.
+package pkg
+
+// Version is the same as the module version.
+const Version = "v1.2.0"
diff --git a/internal/proxy/testdata/single@v1.0.0.txtar b/internal/proxy/testdata/single@v1.0.0.txtar
new file mode 100644
index 0000000..41b0485
--- /dev/null
+++ b/internal/proxy/testdata/single@v1.0.0.txtar
@@ -0,0 +1,56 @@
+A module with a single package that is below the module root.
+
+-- go.mod --
+module example.com/single
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- pkg/file1.go --
+// Package pkg is a sample package.
+package pkg
+
+import "time"
+
+// Version is the same as the module version.
+const Version = "v1.0.0"
+
+// F is a function.
+func F(t time.Time, s string) (T, u) {
+ x := 3
+ x = C
+}
+
+// G is new in v1.1.0.
+func G() int {
+ return 3
+}
+
+-- pkg/file2.go --
+package pkg
+
+var V = Version
+
+type T int
+
+type u int
+
+-- pkg/example_test.go --
+package pkg_test
+
+// Example for the package.
+func Example() {
+ fmt.Println("hello")
+ // Output: hello
+}
+
+// A function example.
+func ExampleF() {
+ pkg.F()
+}
+
+
+
diff --git a/internal/proxy/testdata/symbols@v1.0.0.txtar b/internal/proxy/testdata/symbols@v1.0.0.txtar
new file mode 100644
index 0000000..97b4590
--- /dev/null
+++ b/internal/proxy/testdata/symbols@v1.0.0.txtar
@@ -0,0 +1,84 @@
+A module used for testing the symbols logic.
+
+-- go.mod --
+module example.com/symbols
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- symbols.go --
+package symbols
+
+// const
+const C = 1
+
+// const iota
+const (
+ AA = iota + 1
+ _
+ BB
+ CC
+)
+
+type Num int
+
+const (
+ DD Num = iota
+ _
+ EE
+ FF
+)
+
+// var
+var V = 2
+
+// Multiple variables on the same line.
+var A, B string
+
+// func
+func F() {}
+
+// type
+type T int
+
+// typeConstant
+const CT T = 3
+
+// typeVariable
+var VT T
+
+// multi-line var
+var (
+ ErrA = errors.New("error A")
+ ErrB = errors.New("error B")
+)
+
+// typeFunc
+func TF() T { return T(0) }
+
+// method
+// BUG(uid): this verifies that notes are rendered
+func (T) M() {}
+
+type S1 struct {
+ F int // field
+}
+
+type S2 struct {
+ S1 // embedded struct; should have an id
+}
+
+type I1 interface {
+ M1()
+}
+
+type I2 interface {
+ I1 // embedded interface; should not have an id
+}
+
+type (
+ Int int
+)
diff --git a/internal/proxy/testdata/symbols@v1.1.0.txtar b/internal/proxy/testdata/symbols@v1.1.0.txtar
new file mode 100644
index 0000000..94ed99e
--- /dev/null
+++ b/internal/proxy/testdata/symbols@v1.1.0.txtar
@@ -0,0 +1,149 @@
+A module used for testing the symbols logic.
+
+-- go.mod --
+module example.com/symbols
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- symbols.go --
+package symbols
+
+// const
+const C = 1
+
+// const iota
+const (
+ AA = iota + 1
+ _
+ BB
+ CC
+)
+
+type Num int
+
+const (
+ DD Num = iota
+ _
+ EE
+ FF
+)
+
+// var
+var V = 2
+
+// Multiple variables on the same line.
+var A, B string
+
+// func
+func F() {}
+
+// type
+type T int
+
+// typeConstant
+const CT T = 3
+
+// typeVariable
+var VT T
+
+// multi-line var
+var (
+ ErrA = errors.New("error A")
+ ErrB = errors.New("error B")
+)
+
+// typeFunc
+func TF() T { return T(0) }
+
+// method
+// BUG(uid): this verifies that notes are rendered
+func (T) M() {}
+
+type S1 struct {
+ F int // field
+}
+
+type S2 struct {
+ S1 // embedded struct; should have an id
+ G int
+}
+
+type I1 interface {
+ M1()
+}
+
+type I2 interface {
+ I1 // embedded interface; should not have an id
+ M2()
+}
+
+type (
+ Int int
+ String bool
+)
+
+-- hello/hello.go --
+// +build linux darwin
+// +build amd64
+
+package hello
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello"
+}
+
+-- hello/hello_js.go --
+// +build js,wasm
+
+package hello
+
+// HelloJS returns a greeting when the build context is js/wasm.
+func HelloJS() string {
+ return "Hello"
+}
+
+-- multigoos/multigoos.go --
+// +build darwin linux windows
+
+package multigoos
+
+// type FD is introduced for windows, linux and darwin at this version.
+type FD struct {}
+
+-- multigoos/multigoos_windows.go --
+// +build windows
+
+package multigoos
+
+// Different signature from CloseOnExec for linux and darwin.
+func CloseOnExec(foo string) error {
+ return nil
+}
+
+-- multigoos/multigoos_unix.go --
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package multigoos
+
+// Different signature from CloseOnExec for windows.
+func CloseOnExec(num int) (int, error) {
+ return num, nil
+}
+
+-- duplicate/duplicate.go --
+// +build linux darwin
+
+package duplicate
+
+// Unexported here, exported in v1.2.0.
+type tokenType int
+
+// Token types.
+const (
+ TokenShort tokenType = iota
+)
diff --git a/internal/proxy/testdata/symbols@v1.2.0.txtar b/internal/proxy/testdata/symbols@v1.2.0.txtar
new file mode 100644
index 0000000..5de7c25
--- /dev/null
+++ b/internal/proxy/testdata/symbols@v1.2.0.txtar
@@ -0,0 +1,176 @@
+A module used for testing the symbols logic.
+
+-- go.mod --
+module example.com/symbols
+
+-- README.md --
+This is the README for a test module.
+
+-- LICENSE --
+$MITLicense
+
+-- symbols.go --
+package symbols
+
+// const
+const C = 1
+
+// const iota
+const (
+ AA = iota + 1
+ _
+ BB
+ CC
+)
+
+type Num int
+
+const (
+ DD Num = iota
+ _
+ EE
+ FF
+)
+
+// var
+var V = 2
+
+// Multiple variables on the same line.
+var A, B string
+
+// func
+func F() {}
+
+// type
+type T int
+
+// typeConstant
+const CT T = 3
+
+// typeVariable
+var VT T
+
+// multi-line var
+var (
+ ErrA = errors.New("error A")
+ ErrB = errors.New("error B")
+)
+
+// typeFunc
+func TF() T { return T(0) }
+
+// method
+// BUG(uid): this verifies that notes are rendered
+func (T) M() {}
+
+type S1 struct {
+ F int // field
+}
+
+type S2 struct {
+ S1 // embedded struct; should have an id
+ G int
+}
+
+type I1 interface {
+ M1()
+}
+
+type I2 interface {
+ I1 // embedded interface; should not have an id
+ M2()
+}
+
+type (
+ Int int
+ String bool
+)
+
+-- hello/hello.go --
+package hello
+
+// Hello returns a greeting.
+func Hello() string {
+ return "Hello"
+}
+
+-- hello/hello_js.go --
+// +build js,wasm
+
+package hello
+
+// HelloJS returns a greeting when the build context is js/wasm.
+func HelloJS() string {
+ return "Hello"
+}
+
+-- multigoos/multigoos_windows.go --
+// +build windows
+
+package multigoos
+
+func CloseOnExec(foo string) error {
+ return nil
+}
+
+type FD struct {}
+
+// FD was introduced in v1.1.0 for linux, darwin and windows.
+// MyWindowsMethod is introduced only for windows in this version.
+func (*FD) MyWindowsMethod() {
+}
+
+-- multigoos/multigoos_unix.go --
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package multigoos
+
+func CloseOnExec(num int) (int, error) {
+ return num, nil
+}
+
+type FD struct {}
+
+// FD was introduced in v1.1.0 for linux, darwin and windows.
+// MyMethod is introduced only for darwin and linux in this version.
+func (*FD) MyMethod() {
+}
+
+-- multigoos/multigoos_js.go --
+// +build js,wasm
+
+package multigoos
+
+func CloseOnExec(n int) {
+}
+
+-- duplicate/duplicate.go --
+// +build linux darwin
+
+package duplicate
+
+type TokenType int
+
+// Token types.
+const (
+ TokenShort TokenType = iota
+)
+
+-- duplicate/duplicate_windows.go --
+// +build windows
+
+package duplicate
+
+// Constant here, type for JS, linux and darwin.
+const TokenType = 3
+
+-- duplicate/duplicate_js.go --
+// +build js
+
+package duplicate
+
+// Exported here, unexported in v1.1.0.
+type TokenType struct {
+}
+
+func TokenShort() TokenType { return &TokenType{} }
diff --git a/internal/testing/testhelper/testhelper.go b/internal/testing/testhelper/testhelper.go
new file mode 100644
index 0000000..8f3cb23
--- /dev/null
+++ b/internal/testing/testhelper/testhelper.go
@@ -0,0 +1,176 @@
+// DO NOT EDIT. This file was copied from
+// https://go.googlesource.com/pkgsite/+/035bfc02/internal/testing
+
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package testhelper provides shared functionality and constants to be used in
+// Discovery tests. It should only be imported by test files.
+package testhelper
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+)
+
+const (
+ // MITLicense is the contents of the MIT license. It is detectable by the
+ // licensecheck package, and is considered redistributable.
+ MITLicense = `Copyright 2019 Google Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.`
+
+ // BSD0License is the contents of the BSD-0-Clause license. It is detectable
+ // by the licensecheck package, and is considered redistributable.
+ BSD0License = `Copyright 2019 Google Inc
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.`
+
+ // UnknownLicense is not detectable by the licensecheck package.
+ UnknownLicense = `THIS IS A LICENSE THAT I JUST MADE UP. YOU CAN DO WHATEVER YOU WANT WITH THIS CODE, TRUST ME.`
+)
+
+// SetupTestClientAndServer returns a *httpClient that can be used to
+// stub requests to remote hosts by redirecting all requests that the client
+// makes to a httptest.Server. with the given handler. It also disables TLS
+// verification.
+func SetupTestClientAndServer(handler http.Handler) (*http.Client, *httptest.Server, func()) {
+ srv := httptest.NewTLSServer(handler)
+
+ cli := &http.Client{
+ Transport: &http.Transport{
+ Proxy: http.ProxyFromEnvironment,
+ DialContext: func(_ context.Context, network, _ string) (net.Conn, error) {
+ return net.Dial(network, srv.Listener.Addr().String())
+ },
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: true,
+ },
+ },
+ }
+
+ return cli, srv, srv.Close
+}
+
+func writeZip(w io.Writer, contents map[string]string) (err error) {
+ zw := zip.NewWriter(w)
+ defer func() {
+ if cerr := zw.Close(); cerr != nil {
+ err = fmt.Errorf("error: %v, close error: %v", err, cerr)
+ }
+ }()
+
+ for name, content := range contents {
+ fw, err := zw.Create(name)
+ if err != nil {
+ return fmt.Errorf("ZipWriter::Create(): %v", err)
+ }
+ _, err = io.WriteString(fw, content)
+ if err != nil {
+ return fmt.Errorf("io.WriteString(...): %v", err)
+ }
+ }
+ return nil
+}
+
+// ZipContents creates an in-memory zip of the given contents.
+func ZipContents(contents map[string]string) ([]byte, error) {
+ bs := &bytes.Buffer{}
+ if err := writeZip(bs, contents); err != nil {
+ return nil, fmt.Errorf("testhelper.ZipContents(%v): %v", contents, err)
+ }
+ return bs.Bytes(), nil
+}
+
+// TestDataPath returns a path corresponding to a path relative to the calling
+// test file. For convenience, rel is assumed to be "/"-delimited.
+//
+// It panics on failure.
+func TestDataPath(rel string) string {
+ _, filename, _, ok := runtime.Caller(1)
+ if !ok {
+ panic("unable to determine relative path")
+ }
+ return filepath.Clean(filepath.Join(filepath.Dir(filename), filepath.FromSlash(rel)))
+}
+
+// CreateTestDirectory creates a directory to hold a module when testing
+// local fetching, and returns the directory.
+func CreateTestDirectory(files map[string]string) (_ string, err error) {
+ defer derrors.Wrap(&err, "CreateTestDirectory(files)")
+ tempDir, err := os.MkdirTemp("", "")
+ if err != nil {
+ return "", err
+ }
+
+ for path, contents := range files {
+ path = filepath.Join(tempDir, path)
+
+ parent, _ := filepath.Split(path)
+ if err := os.MkdirAll(parent, 0755); err != nil {
+ return "", err
+ }
+
+ file, err := os.Create(path)
+ if err != nil {
+ return "", err
+ }
+ if _, err := file.WriteString(contents); err != nil {
+ return "", err
+ }
+ if err := file.Close(); err != nil {
+ return "", err
+ }
+ }
+
+ return tempDir, nil
+}
+
+func CompareWithGolden(t *testing.T, got, filename string, update bool) {
+ t.Helper()
+ if update {
+ writeGolden(t, filename, got)
+ } else {
+ want := readGolden(t, filename)
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("%s: mismatch (-want, +got):\n%s", filename, diff)
+ }
+ }
+}
+
+func writeGolden(t *testing.T, name string, data string) {
+ filename := filepath.Join("testdata", name)
+ if err := os.WriteFile(filename, []byte(data), 0644); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("wrote %s", filename)
+}
+
+func readGolden(t *testing.T, name string) string {
+ data, err := os.ReadFile(filepath.Join("testdata", name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(data)
+}
