cmd/pkgsite: serve modules from the build list
Instead of serving just the module at a given path, serve all the
modules on its build list too (the result of go list -m all).
Change-Id: Ibd899d0a7bf23abd3e81882c2a1a62f2f63570a2
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/386194
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index af8ebba..43e2786 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -13,6 +13,10 @@
//
// cd ~/repos/cue && pkgsite
//
+// This form will also serve all of the module's required modules at their
+// required versions. You can disable serving the required modules by passing
+// -list=false.
+//
// You can also serve docs from your module cache, directly from the proxy
// (it uses the GOPROXY environment variable), or both:
//
@@ -31,12 +35,15 @@
package main
import (
+ "bytes"
"context"
+ "encoding/json"
"flag"
"fmt"
"net/http"
"os"
"os/exec"
+ "path/filepath"
"strings"
"time"
@@ -57,12 +64,13 @@
const defaultAddr = "localhost:8080" // default webserver address
var (
- gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
- httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
- useCache = flag.Bool("cache", false, "fetch from the module cache")
- cacheDir = flag.String("cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
- useProxy = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
- goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem")
+ gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
+ httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
+ useCache = flag.Bool("cache", false, "fetch from the module cache")
+ cacheDir = flag.String("cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
+ useProxy = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
+ goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem")
+ useListedMods = flag.Bool("list", true, "for each path, serve all modules in build list")
)
func main() {
@@ -82,7 +90,7 @@
}
var modCacheDir string
- if *useCache {
+ if *useCache || *useListedMods {
modCacheDir = *cacheDir
if modCacheDir == "" {
var err error
@@ -99,6 +107,7 @@
if *useCache || *useProxy {
fmt.Fprintf(os.Stderr, "BYPASSING LICENSE CHECKING: MAY DISPLAY NON-REDISTRIBUTABLE INFORMATION\n")
}
+
var prox *proxy.Client
if *useProxy {
url := os.Getenv("GOPROXY")
@@ -116,7 +125,16 @@
stdlib.SetGoRepoPath(*goRepoPath)
}
- server, err := newServer(ctx, paths, *gopathMode, modCacheDir, prox)
+ var cacheMods []internal.Modver
+ if *useListedMods && !*useCache {
+ var err error
+ paths, cacheMods, err = listModsForPaths(paths, modCacheDir)
+ if err != nil {
+ die("listing mods (consider passing -list=false): %v", err)
+ }
+ }
+
+ server, err := newServer(ctx, paths, *gopathMode, modCacheDir, cacheMods, prox)
if err != nil {
die("%s", err)
}
@@ -141,10 +159,10 @@
return paths
}
-func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir string, prox *proxy.Client) (*frontend.Server, error) {
+func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir string, cacheMods []internal.Modver, prox *proxy.Client) (*frontend.Server, error) {
getters := buildGetters(ctx, paths, gopathMode)
if downloadDir != "" {
- g, err := fetch.NewFSProxyModuleGetter(downloadDir)
+ g, err := fetch.NewFSProxyModuleGetter(downloadDir, cacheMods)
if err != nil {
return nil, err
}
@@ -204,9 +222,63 @@
}
func defaultCacheDir() (string, error) {
- out, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput()
+ out, err := runGo("", "env", "GOMODCACHE")
if err != nil {
- return "", fmt.Errorf("running 'go env GOMODCACHE': %v: %s", err, out)
+ return "", err
}
return strings.TrimSpace(string(out)), nil
}
+
+// listedMod has a subset of the fields written by `go list -m`.
+type listedMod struct {
+ internal.Modver
+ GoMod string // absolute path to go.mod file; in download cache or replaced
+}
+
+var listModules = _listModules
+
+func _listModules(dir string) ([]listedMod, error) {
+ out, err := runGo(dir, "list", "-json", "-m", "all")
+ if err != nil {
+ return nil, err
+ }
+ d := json.NewDecoder(bytes.NewReader(out))
+ var ms []listedMod
+ for d.More() {
+ var m listedMod
+ if err := d.Decode(&m); err != nil {
+ return nil, err
+ }
+ ms = append(ms, m)
+ }
+ return ms, nil
+}
+
+func runGo(dir string, args ...string) ([]byte, error) {
+ cmd := exec.Command("go", args...)
+ cmd.Dir = dir
+ out, err := cmd.Output()
+ if err != nil {
+ return nil, fmt.Errorf("running go with %q: %v: %s", args, err, out)
+ }
+ return out, nil
+}
+
+func listModsForPaths(paths []string, cacheDir string) ([]string, []internal.Modver, error) {
+ var outPaths []string
+ var cacheMods []internal.Modver
+ for _, p := range paths {
+ lms, err := listModules(p)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, lm := range lms {
+ if strings.HasPrefix(lm.GoMod, cacheDir) {
+ cacheMods = append(cacheMods, lm.Modver)
+ } else { // probably the result of a replace directive
+ outPaths = append(outPaths, filepath.Dir(lm.GoMod))
+ }
+ }
+ }
+ return outPaths, cacheMods, nil
+}
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index 9a84ee4..86d5cab 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -16,6 +16,7 @@
"github.com/google/go-cmp/cmp"
"golang.org/x/net/html"
+ "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/proxy/proxytest"
"golang.org/x/pkgsite/internal/testing/htmlcheck"
)
@@ -48,7 +49,7 @@
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()
- server, err := newServer(context.Background(), []string{localModule}, false, cacheDir, prox)
+ server, err := newServer(context.Background(), []string{localModule}, false, cacheDir, nil, prox)
if err != nil {
t.Fatal(err)
}
@@ -122,3 +123,29 @@
t.Errorf("got %v, want %v", got, want)
}
}
+
+func TestListModsForPaths(t *testing.T) {
+ listModules = func(string) ([]listedMod, error) {
+ return []listedMod{
+ {
+ internal.Modver{Path: "m1", Version: "v1.2.3"},
+ "/dir/cache/download/m1/@v/v1.2.3.mod",
+ },
+ {
+ internal.Modver{Path: "m2", Version: "v1.0.0"},
+ "/repos/m2/go.mod",
+ },
+ }, nil
+ }
+ defer func() { listModules = _listModules }()
+
+ gotPaths, gotCacheMods, err := listModsForPaths([]string{"m1"}, "/dir")
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantPaths := []string{"/repos/m2"}
+ wantCacheMods := []internal.Modver{{Path: "m1", Version: "v1.2.3"}}
+ if !cmp.Equal(gotPaths, wantPaths) || !cmp.Equal(gotCacheMods, wantCacheMods) {
+ t.Errorf("got\n%v, %v\nwant\n%v, %v", gotPaths, gotCacheMods, wantPaths, wantCacheMods)
+ }
+}
diff --git a/internal/fetch/getters.go b/internal/fetch/getters.go
index e02d900..4fb74ef 100644
--- a/internal/fetch/getters.go
+++ b/internal/fetch/getters.go
@@ -23,6 +23,7 @@
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
+ "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/source"
@@ -194,19 +195,29 @@
// paths that correspond to proxy URLs. An example of such a directory is
// $(go env GOMODCACHE).
type fsProxyModuleGetter struct {
- dir string
+ dir string
+ allowed map[internal.Modver]bool
}
// NewFSModuleGetter return a ModuleGetter that reads modules from a filesystem
// directory organized like the proxy.
-func NewFSProxyModuleGetter(dir string) (_ *fsProxyModuleGetter, err error) {
+// If allowed is non-empty, only module@versions in allowed are served; others
+// result in NotFound errors.
+func NewFSProxyModuleGetter(dir string, allowed []internal.Modver) (_ *fsProxyModuleGetter, err error) {
defer derrors.Wrap(&err, "NewFSProxyModuleGetter(%q)", dir)
abs, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
- return &fsProxyModuleGetter{dir: abs}, nil
+ g := &fsProxyModuleGetter{dir: abs}
+ if len(allowed) > 0 {
+ g.allowed = map[internal.Modver]bool{}
+ for _, a := range allowed {
+ g.allowed[a] = true
+ }
+ }
+ return g, nil
}
// Info returns basic information about the module.
@@ -248,7 +259,7 @@
}
}
- // Check that .zip is readable first.
+ // Check that .zip is readable.
f, err := g.openFile(path, vers, "zip")
if err != nil {
return nil, err
@@ -267,6 +278,7 @@
return nil, err
}
}
+
data, err := g.readFile(path, vers, "zip")
if err != nil {
return nil, err
@@ -307,14 +319,29 @@
}
var versions []string
for _, z := range zips {
- versions = append(versions, strings.TrimSuffix(filepath.Base(z), ".zip"))
+ vers := strings.TrimSuffix(filepath.Base(z), ".zip")
+ if g.allow(modulePath, vers) {
+ versions = append(versions, vers)
+ }
+ }
+ if len(versions) == 0 {
+ return "", fmt.Errorf("no allowed versions form module %q: %w", modulePath, derrors.NotFound)
}
return version.LatestOf(versions), nil
}
+// allow reports whether the module path and version may be served.
+func (g *fsProxyModuleGetter) allow(path, vers string) bool {
+ return g.allowed == nil || g.allowed[internal.Modver{Path: path, Version: vers}]
+}
+
func (g *fsProxyModuleGetter) readFile(path, version, suffix string) (_ []byte, err error) {
defer derrors.Wrap(&err, "fsProxyModuleGetter.readFile(%q, %q, %q)", path, version, suffix)
+ if !g.allow(path, version) {
+ return nil, fmt.Errorf("%s@%s not allowed: %w", path, version, derrors.NotFound)
+ }
+
f, err := g.openFile(path, version, suffix)
if err != nil {
return nil, err
diff --git a/internal/fetch/getters_test.go b/internal/fetch/getters_test.go
index 92c47ff..5d3dbb8 100644
--- a/internal/fetch/getters_test.go
+++ b/internal/fetch/getters_test.go
@@ -13,6 +13,7 @@
"time"
"github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/version"
@@ -47,7 +48,7 @@
"dir/cache/download/github.com/a!bc/@v/v2.3.4.zip",
},
} {
- g, err := NewFSProxyModuleGetter("dir")
+ g, err := NewFSProxyModuleGetter("dir", nil)
if err != nil {
t.Fatal(err)
}
@@ -76,7 +77,7 @@
if err != nil {
t.Fatal(err)
}
- g, err := NewFSProxyModuleGetter("testdata/modcache")
+ g, err := NewFSProxyModuleGetter("testdata/modcache", nil)
if err != nil {
t.Fatal(err)
}
@@ -142,3 +143,37 @@
}
})
}
+
+func TestFSProxyGetterAllowed(t *testing.T) {
+ ctx := context.Background()
+ // The cache contains:
+ // github.com/jackc/pgio@v1.0.0
+ // modcache.com@v1.0.0
+ // Allow only the latter.
+ allow := []internal.Modver{{Path: "modcache.com", Version: "v1.0.0"}}
+ g, err := NewFSProxyModuleGetter("testdata/modcache", allow)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ check := func(path, vers string, wantOK bool) {
+ t.Helper()
+ _, err := g.Info(ctx, path, vers)
+ if (err == nil) != wantOK {
+ s := "a"
+ if wantOK {
+ s = "no"
+ }
+ t.Errorf("got %v; wanted %s error", err, s)
+ } else if err != nil && !errors.Is(err, derrors.NotFound) {
+ t.Errorf("got %v, wanted NotFound", err)
+ }
+ }
+
+ // We can get modcache.com.
+ check("modcache.com", "v1.0.0", true)
+ check("modcache.com", "latest", true)
+ // We get NotFound for jackc.
+ check("github.com/jackc/pgio", "v1.0.0", false)
+ check("github.com/jackc/pgio", "latest", false)
+}
