internal/{config,proxy,worker}: use proxy cached-only URL
Use the new "/cached-only" suffix to the proxy URL throughout
the worker, to avoid asking the proxy to download new code.
See https://proxy.golang.org, under "Services".
Change-Id: Icddcafb92486d6a9bd1a2490dfb2b959bf0b4e80
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/496655
Reviewed-by: Zvonimir Pavlinovic <zpavlinovic@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
diff --git a/internal/config/config.go b/internal/config/config.go
index 7c3af0e..af4d972 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -140,7 +140,7 @@
PkgsiteDBName: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_NAME", "discovery-db"),
PkgsiteDBUser: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_USER", "postgres"),
PkgsiteDBSecret: os.Getenv("GO_ECOSYSTEM_PKGSITE_DB_SECRET"),
- ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"),
+ ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org/cached-only"),
VulnDBURL: GetEnv("GO_VULNDB_URL", "https://vuln.go.dev"),
}
if OnCloudRun() {
diff --git a/internal/proxy/server.go b/internal/proxy/server.go
deleted file mode 100644
index 3d6fc4a..0000000
--- a/internal/proxy/server.go
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2023 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proxy
-
-import (
- "io"
- "net/http"
- "net/http/httptest"
-
- "golang.org/x/pkgsite-metrics/internal/log"
-)
-
-// ServeDisablingFetch returns a server that proxies requests to proxy.golang.org,
-// adding the Disable-Fetch header to prevent the proxy from fetching old modules.
-func ServeDisablingFetch() *httptest.Server {
- return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- resp, err := proxyRequest(r)
- if err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return
- }
- for k, vs := range resp.Header {
- for _, v := range vs {
- w.Header().Add(k, v)
- }
- }
- w.WriteHeader(resp.StatusCode)
- defer resp.Body.Close()
- _, err = io.Copy(w, resp.Body)
- if err != nil {
- log.Errorf(r.Context(), err, "ServeDisablingFetch: io.Copy")
- }
- }))
-}
-
-func proxyRequest(r *http.Request) (*http.Response, error) {
- url := "https://proxy.golang.org" + r.URL.Path
- req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, url, nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set(DisableFetchHeader, "true")
- return http.DefaultClient.Do(req)
-}
diff --git a/internal/proxy/server_test.go b/internal/proxy/server_test.go
deleted file mode 100644
index edf6248..0000000
--- a/internal/proxy/server_test.go
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2023 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proxy
-
-import (
- "bytes"
- "io"
- "net/http"
- "net/url"
- "testing"
-
- test "golang.org/x/pkgsite-metrics/internal/testing"
-)
-
-// This module is currently not stored on the proxy.
-const missingModule = "code.haiziwang.com/golang/autil"
-
-func TestServer(t *testing.T) {
- test.NeedsIntegrationEnv(t)
- s := ServeDisablingFetch()
- defer s.Close()
- serverURL, err := url.Parse(s.URL)
- if err != nil {
- t.Fatal(err)
- }
-
- url := serverURL.JoinPath("github.com/pkg/errors/@v/list")
- resp, err := http.Get(url.String())
- if err != nil {
- t.Fatal(err)
- }
- if g, w := resp.StatusCode, 200; g != w {
- t.Fatalf("status: got %d, want %d", g, w)
- }
- defer resp.Body.Close()
- got, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatal(err)
- }
- want := "v0.8.1\n"
- if !bytes.Contains(got, []byte(want)) {
- t.Errorf("got body %q, missing %q", got, want)
- }
-
- url = serverURL.JoinPath(missingModule, "/@v/list")
- resp, err = http.Get(url.String())
- if err != nil {
- t.Fatal(err)
- }
- if g, w := resp.StatusCode, http.StatusNotFound; g != w {
- t.Fatalf("status: got %d, want %d", g, w)
- }
-}
diff --git a/internal/worker/scan.go b/internal/worker/scan.go
index 2cccd05..f2b08bf 100644
--- a/internal/worker/scan.go
+++ b/internal/worker/scan.go
@@ -233,9 +233,8 @@
// Download all dependencies, using the given directory for the Go module cache
// if it is non-empty.
opts := &goCommandOptions{
- dir: dir,
- insecure: insecure,
- proxyAddr: "https://proxy.golang.org",
+ dir: dir,
+ insecure: insecure,
}
return runGoCommand(ctx, modulePath, version, opts, "mod", "download")
}
@@ -257,20 +256,16 @@
// goModTidy runs "go mod tidy" on a module in dir.
func goModTidy(ctx context.Context, modulePath, version, dir string, insecure bool) error {
- s := proxy.ServeDisablingFetch()
- defer s.Close()
opts := &goCommandOptions{
- dir: dir,
- insecure: insecure,
- proxyAddr: s.URL,
+ dir: dir,
+ insecure: insecure,
}
return runGoCommand(ctx, modulePath, version, opts, "mod", "tidy")
}
type goCommandOptions struct {
- dir string
- proxyAddr string
- insecure bool
+ dir string
+ insecure bool
}
// runGoModCommand runs the command `go args...`.
@@ -286,9 +281,7 @@
cmd := exec.Command("go", args...)
cmd.Dir = opts.dir
cmd.Env = cmd.Environ()
- if opts.proxyAddr != "" {
- cmd.Env = append(cmd.Env, fmt.Sprintf("GOPROXY=%s", opts.proxyAddr))
- }
+ cmd.Env = append(cmd.Env, "GOPROXY=https://proxy.golang.org/cached-only")
if !opts.insecure {
// Use sandbox mod cache.
cmd.Env = append(cmd.Env, "GOMODCACHE="+filepath.Join(sandboxRoot, sandboxGoModCache))
diff --git a/internal/worker/scan_test.go b/internal/worker/scan_test.go
index 69cad4c..4233298 100644
--- a/internal/worker/scan_test.go
+++ b/internal/worker/scan_test.go
@@ -23,7 +23,7 @@
ctx := context.Background()
slog.SetDefault(slog.New(log.NewLineHandler(os.Stderr)))
const insecure = true
- proxyClient, err := proxy.New("https://proxy.golang.org")
+ proxyClient, err := proxy.New("https://proxy.golang.org/cached-only")
if err != nil {
t.Fatal(err)
}
