cmd/go/internal/web: merge internal/web2 into web
The cmd/go/internal/web package was forked in order to support direct
HTTPS fetches from widely-used hosting providers,¹ but direct fetches
were subsequently dropped in CL 107657. The forked web2 package, with
its GitHub-specific diagnostics and .netrc support, remained in use
for module proxy support, but was not used for the initial '?go-get=1'
path resolution, so the .netrc file was only used to fetch from
already-resolved module protocol servers.
This CL moves the .netrc support into its own (new) package,
cmd/go/internal/auth, and consolidates the web and web2 packages back
into just web. As a result, fetches via the web package now support
.netrc, and fetches that previously used web2 now enforce the same
security policies as web (such as prohibiting HTTPS-to-HTTP
redirects).
¹https://github.com/golang/vgo/commit/63138cb6ceed7d6d4e51a8cbd568c64bd3e2b132
Fixes #29591
Fixes #29888
Fixes #30610
Updates #26232
Change-Id: Ia3a13526e443679cf14a72a1f3db96f336ce5e73
Reviewed-on: https://go-review.googlesource.com/c/go/+/170879
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
diff --git a/src/cmd/go/internal/auth/auth.go b/src/cmd/go/internal/auth/auth.go
new file mode 100644
index 0000000..12e3c74
--- /dev/null
+++ b/src/cmd/go/internal/auth/auth.go
@@ -0,0 +1,23 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package auth provides access to user-provided authentication credentials.
+package auth
+
+import "net/http"
+
+// AddCredentials fills in the user's credentials for req, if any.
+// The return value reports whether any matching credentials were found.
+func AddCredentials(req *http.Request) (added bool) {
+ // TODO(golang.org/issue/26232): Support arbitrary user-provided credentials.
+ netrcOnce.Do(readNetrc)
+ for _, l := range netrc {
+ if l.machine == req.URL.Host {
+ req.SetBasicAuth(l.login, l.password)
+ return true
+ }
+ }
+
+ return false
+}
diff --git a/src/cmd/go/internal/auth/netrc.go b/src/cmd/go/internal/auth/netrc.go
new file mode 100644
index 0000000..7a9bdbb
--- /dev/null
+++ b/src/cmd/go/internal/auth/netrc.go
@@ -0,0 +1,111 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "sync"
+)
+
+type netrcLine struct {
+ machine string
+ login string
+ password string
+}
+
+var (
+ netrcOnce sync.Once
+ netrc []netrcLine
+ netrcErr error
+)
+
+func parseNetrc(data string) []netrcLine {
+ // See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
+ // for documentation on the .netrc format.
+ var nrc []netrcLine
+ var l netrcLine
+ inMacro := false
+ for _, line := range strings.Split(data, "\n") {
+ if inMacro {
+ if line == "" {
+ inMacro = false
+ }
+ continue
+ }
+
+ f := strings.Fields(line)
+ i := 0
+ for ; i < len(f)-1; i += 2 {
+ // Reset at each "machine" token.
+ // “The auto-login process searches the .netrc file for a machine token
+ // that matches […]. Once a match is made, the subsequent .netrc tokens
+ // are processed, stopping when the end of file is reached or another
+ // machine or a default token is encountered.”
+ switch f[i] {
+ case "machine":
+ l = netrcLine{machine: f[i+1]}
+ case "default":
+ break
+ case "login":
+ l.login = f[i+1]
+ case "password":
+ l.password = f[i+1]
+ case "macdef":
+ // “A macro is defined with the specified name; its contents begin with
+ // the next .netrc line and continue until a null line (consecutive
+ // new-line characters) is encountered.”
+ inMacro = true
+ }
+ if l.machine != "" && l.login != "" && l.password != "" {
+ nrc = append(nrc, l)
+ l = netrcLine{}
+ }
+ }
+
+ if i < len(f) && f[i] == "default" {
+ // “There can be only one default token, and it must be after all machine tokens.”
+ break
+ }
+ }
+
+ return nrc
+}
+
+func netrcPath() (string, error) {
+ if env := os.Getenv("NETRC"); env != "" {
+ return env, nil
+ }
+ dir, err := os.UserHomeDir()
+ if err != nil {
+ return "", err
+ }
+ base := ".netrc"
+ if runtime.GOOS == "windows" {
+ base = "_netrc"
+ }
+ return filepath.Join(dir, base), nil
+}
+
+func readNetrc() {
+ path, err := netrcPath()
+ if err != nil {
+ netrcErr = err
+ return
+ }
+
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ netrcErr = err
+ }
+ return
+ }
+
+ netrc = parseNetrc(string(data))
+}
diff --git a/src/cmd/go/internal/web2/web_test.go b/src/cmd/go/internal/auth/netrc_test.go
similarity index 94%
rename from src/cmd/go/internal/web2/web_test.go
rename to src/cmd/go/internal/auth/netrc_test.go
index e6787a5..e06c545 100644
--- a/src/cmd/go/internal/web2/web_test.go
+++ b/src/cmd/go/internal/auth/netrc_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package web2
+package auth
import (
"reflect"
@@ -43,7 +43,7 @@
password too-late-in-file
`
-func TestReadNetrc(t *testing.T) {
+func TestParseNetrc(t *testing.T) {
lines := parseNetrc(testNetrc)
want := []netrcLine{
{"api.github.com", "user", "pwd"},
diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go
index e701f6e..468605c 100644
--- a/src/cmd/go/internal/bug/bug.go
+++ b/src/cmd/go/internal/bug/bug.go
@@ -10,6 +10,7 @@
"fmt"
"io"
"io/ioutil"
+ urlpkg "net/url"
"os"
"os/exec"
"path/filepath"
@@ -62,7 +63,7 @@
fmt.Fprintln(&buf, "```")
body := buf.String()
- url := "https://github.com/golang/go/issues/new?body=" + web.QueryEscape(body)
+ url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body)
if !web.OpenBrowser(url) {
fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n")
fmt.Print(body)
@@ -130,7 +131,12 @@
}
func inspectGoVersion(w io.Writer) {
- data, err := web.Get("https://golang.org/VERSION?m=text")
+ data, err := web.GetBytes(&urlpkg.URL{
+ Scheme: "https",
+ Host: "golang.org",
+ Path: "/VERSION",
+ RawQuery: "?m=text",
+ })
if err != nil {
if cfg.BuildV {
fmt.Printf("failed to read from golang.org/VERSION: %v\n", err)
diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go
index fe15515..c70013c 100644
--- a/src/cmd/go/internal/get/get.go
+++ b/src/cmd/go/internal/get/get.go
@@ -392,7 +392,7 @@
blindRepo bool // set if the repo has unusual configuration
)
- security := web.Secure
+ security := web.SecureOnly
if Insecure {
security = web.Insecure
}
diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/get/vcs.go
index 6f60bc0..bb1845e 100644
--- a/src/cmd/go/internal/get/vcs.go
+++ b/src/cmd/go/internal/get/vcs.go
@@ -11,7 +11,7 @@
"internal/lazyregexp"
"internal/singleflight"
"log"
- "net/url"
+ urlpkg "net/url"
"os"
"os/exec"
"path/filepath"
@@ -54,7 +54,7 @@
}
func (v *vcsCmd) isSecure(repo string) bool {
- u, err := url.Parse(repo)
+ u, err := urlpkg.Parse(repo)
if err != nil {
// If repo is not a URL, it's not secure.
return false
@@ -188,19 +188,19 @@
}
out := strings.TrimSpace(string(outb))
- var repoURL *url.URL
+ var repoURL *urlpkg.URL
if m := scpSyntaxRe.FindStringSubmatch(out); m != nil {
// Match SCP-like syntax and convert it to a URL.
// Eg, "git@github.com:user/repo" becomes
// "ssh://git@github.com/user/repo".
- repoURL = &url.URL{
+ repoURL = &urlpkg.URL{
Scheme: "ssh",
- User: url.User(m[1]),
+ User: urlpkg.User(m[1]),
Host: m[2],
Path: m[3],
}
} else {
- repoURL, err = url.Parse(out)
+ repoURL, err = urlpkg.Parse(out)
if err != nil {
return "", err
}
@@ -730,7 +730,7 @@
match["repo"] = scheme + "://" + match["repo"]
} else {
for _, scheme := range vcs.scheme {
- if security == web.Secure && !vcs.isSecureScheme(scheme) {
+ if security == web.SecureOnly && !vcs.isSecureScheme(scheme) {
continue
}
if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
@@ -754,20 +754,35 @@
return nil, errUnknownSite
}
+// urlForImportPath returns a partially-populated URL for the given Go import path.
+//
+// The URL leaves the Scheme field blank so that web.Get will try any scheme
+// allowed by the selected security mode.
+func urlForImportPath(importPath string) (*urlpkg.URL, error) {
+ slash := strings.Index(importPath, "/")
+ if slash < 0 {
+ slash = len(importPath)
+ }
+ host, path := importPath[:slash], importPath[slash:]
+ if !strings.Contains(host, ".") {
+ return nil, errors.New("import path does not begin with hostname")
+ }
+ if len(path) == 0 {
+ path = "/"
+ }
+ return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil
+}
+
// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
// statically known by repoRootForImportPathStatic.
//
// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
- slash := strings.Index(importPath, "/")
- if slash < 0 {
- slash = len(importPath)
+ url, err := urlForImportPath(importPath)
+ if err != nil {
+ return nil, err
}
- host := importPath[:slash]
- if !strings.Contains(host, ".") {
- return nil, errors.New("import path does not begin with hostname")
- }
- urlStr, body, err := web.GetMaybeInsecure(importPath, security)
+ url, resp, err := web.Get(security, url)
if err != nil {
msg := "https fetch: %v"
if security == web.Insecure {
@@ -775,6 +790,7 @@
}
return nil, fmt.Errorf(msg, err)
}
+ body := resp.Body
defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if err != nil {
@@ -784,12 +800,12 @@
mmi, err := matchGoImport(imports, importPath)
if err != nil {
if _, ok := err.(ImportMismatchError); !ok {
- return nil, fmt.Errorf("parse %s: %v", urlStr, err)
+ return nil, fmt.Errorf("parse %s: %v", url, err)
}
- return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err)
+ return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", url, err)
}
if cfg.BuildV {
- log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr)
+ log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url)
}
// If the import was "uni.edu/bob/project", which said the
// prefix was "uni.edu" and the RepoRoot was "evilroot.com",
@@ -801,24 +817,24 @@
if cfg.BuildV {
log.Printf("get %q: verifying non-authoritative meta tag", importPath)
}
- urlStr0 := urlStr
+ url0 := *url
var imports []metaImport
- urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
+ url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
if err != nil {
return nil, err
}
metaImport2, err := matchGoImport(imports, importPath)
if err != nil || mmi != metaImport2 {
- return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix)
+ return nil, fmt.Errorf("%s and %s disagree about go-import for %s", &url0, url, mmi.Prefix)
}
}
if err := validateRepoRoot(mmi.RepoRoot); err != nil {
- return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err)
+ return nil, fmt.Errorf("%s: invalid repo root %q: %v", url, mmi.RepoRoot, err)
}
vcs := vcsByCmd(mmi.VCS)
if vcs == nil && mmi.VCS != "mod" {
- return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS)
+ return nil, fmt.Errorf("%s: unknown vcs %q", url, mmi.VCS)
}
rr := &RepoRoot{
@@ -834,7 +850,7 @@
// validateRepoRoot returns an error if repoRoot does not seem to be
// a valid URL with scheme.
func validateRepoRoot(repoRoot string) error {
- url, err := url.Parse(repoRoot)
+ url, err := urlpkg.Parse(repoRoot)
if err != nil {
return err
}
@@ -856,9 +872,9 @@
//
// The importPath is of the form "golang.org/x/tools".
// It is an error if no imports are found.
-// urlStr will still be valid if err != nil.
-// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1"
-func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
+// url will still be valid if err != nil.
+// The returned url will be of the form "https://golang.org/x/tools?go-get=1"
+func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) {
setCache := func(res fetchResult) (fetchResult, error) {
fetchCacheMu.Lock()
defer fetchCacheMu.Unlock()
@@ -874,25 +890,31 @@
}
fetchCacheMu.Unlock()
- urlStr, body, err := web.GetMaybeInsecure(importPrefix, security)
+ url, err := urlForImportPath(importPrefix)
if err != nil {
- return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)})
+ return setCache(fetchResult{err: err})
}
+ url, resp, err := web.Get(security, url)
+ if err != nil {
+ return setCache(fetchResult{url: url, err: fmt.Errorf("fetch %s: %v", url, err)})
+ }
+ body := resp.Body
+ defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if err != nil {
- return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)})
+ return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", url, err)})
}
if len(imports) == 0 {
- err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr)
+ err = fmt.Errorf("fetch %s: no go-import meta tag", url)
}
- return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err})
+ return setCache(fetchResult{url: url, imports: imports, err: err})
})
res := resi.(fetchResult)
- return res.urlStr, res.imports, res.err
+ return res.url, res.imports, res.err
}
type fetchResult struct {
- urlStr string // e.g. "https://foo.com/x/bar?go-get=1"
+ url *urlpkg.URL
imports []metaImport
err error
}
@@ -1074,8 +1096,13 @@
var resp struct {
SCM string `json:"scm"`
}
- url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm")
- data, err := web.Get(url)
+ url := &urlpkg.URL{
+ Scheme: "https",
+ Host: "api.bitbucket.org",
+ Path: expand(match, "/2.0/repositories/{bitname}"),
+ RawQuery: "fields=scm",
+ }
+ data, err := web.GetBytes(url)
if err != nil {
if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 {
// this may be a private repository. If so, attempt to determine which
@@ -1117,7 +1144,12 @@
if match["project"] == "" || match["series"] == "" {
return nil
}
- _, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format"))
+ url := &urlpkg.URL{
+ Scheme: "https",
+ Host: "code.launchpad.net",
+ Path: expand(match, "/{project}{series}/.bzr/branch-format"),
+ }
+ _, err := web.GetBytes(url)
if err != nil {
match["root"] = expand(match, "launchpad.net/{project}")
match["repo"] = expand(match, "https://{root}")
diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/get/vcs_test.go
index d13721b..91800ba 100644
--- a/src/cmd/go/internal/get/vcs_test.go
+++ b/src/cmd/go/internal/get/vcs_test.go
@@ -181,7 +181,7 @@
}
for _, test := range tests {
- got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure)
+ got, err := RepoRootForImportPath(test.path, IgnoreMod, web.SecureOnly)
want := test.want
if want == nil {
diff --git a/src/cmd/go/internal/modfetch/noweb.go b/src/cmd/go/internal/modfetch/noweb.go
deleted file mode 100644
index 9d713dc..0000000
--- a/src/cmd/go/internal/modfetch/noweb.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build cmd_go_bootstrap
-
-package modfetch
-
-import (
- "fmt"
- "io"
-)
-
-func webGetGoGet(url string, body *io.ReadCloser) error {
- return fmt.Errorf("no network in go_bootstrap")
-}
-
-func webGetBytes(url string, body *[]byte) error {
- return fmt.Errorf("no network in go_bootstrap")
-}
-
-func webGetBody(url string, body *io.ReadCloser) error {
- return fmt.Errorf("no network in go_bootstrap")
-}
diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go
index cbf476d..ec9caf1 100644
--- a/src/cmd/go/internal/modfetch/proxy.go
+++ b/src/cmd/go/internal/modfetch/proxy.go
@@ -8,7 +8,11 @@
"encoding/json"
"fmt"
"io"
- "net/url"
+ "io/ioutil"
+ urlpkg "net/url"
+ "os"
+ pathpkg "path"
+ "path/filepath"
"strings"
"time"
@@ -17,6 +21,7 @@
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/semver"
+ "cmd/go/internal/web"
)
var HelpGoproxy = &base.Command{
@@ -99,34 +104,85 @@
if strings.Contains(proxyURL, ",") {
return nil, fmt.Errorf("invalid $GOPROXY setting: cannot have comma")
}
- u, err := url.Parse(proxyURL)
- if err != nil || u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "file" {
- // Don't echo $GOPROXY back in case it has user:password in it (sigh).
- return nil, fmt.Errorf("invalid $GOPROXY setting: malformed URL or invalid scheme (must be http, https, file)")
+ r, err := newProxyRepo(proxyURL, path)
+ if err != nil {
+ return nil, err
}
- return newProxyRepo(u.String(), path)
+ return r, nil
}
type proxyRepo struct {
- url string
+ url *urlpkg.URL
path string
}
func newProxyRepo(baseURL, path string) (Repo, error) {
+ url, err := urlpkg.Parse(baseURL)
+ if err != nil {
+ return nil, err
+ }
+ switch url.Scheme {
+ case "file":
+ if *url != (urlpkg.URL{Scheme: url.Scheme, Path: url.Path, RawPath: url.RawPath}) {
+ return nil, fmt.Errorf("proxy URL %q uses file scheme with non-path elements", web.PasswordRedacted(url))
+ }
+ case "http", "https":
+ case "":
+ return nil, fmt.Errorf("proxy URL %q missing scheme", web.PasswordRedacted(url))
+ default:
+ return nil, fmt.Errorf("unsupported proxy scheme %q", url.Scheme)
+ }
+
enc, err := module.EncodePath(path)
if err != nil {
return nil, err
}
- return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc), path}, nil
+
+ url.Path = strings.TrimSuffix(url.Path, "/") + "/" + enc
+ url.RawPath = strings.TrimSuffix(url.RawPath, "/") + "/" + pathEscape(enc)
+ return &proxyRepo{url, path}, nil
}
func (p *proxyRepo) ModulePath() string {
return p.path
}
+func (p *proxyRepo) getBytes(path string) ([]byte, error) {
+ body, err := p.getBody(path)
+ if err != nil {
+ return nil, err
+ }
+ defer body.Close()
+ return ioutil.ReadAll(body)
+}
+
+func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
+ fullPath := pathpkg.Join(p.url.Path, path)
+ if p.url.Scheme == "file" {
+ rawPath, err := urlpkg.PathUnescape(fullPath)
+ if err != nil {
+ return nil, err
+ }
+ return os.Open(filepath.FromSlash(rawPath))
+ }
+
+ url := new(urlpkg.URL)
+ *url = *p.url
+ url.Path = fullPath
+ url.RawPath = pathpkg.Join(url.RawPath, pathEscape(path))
+
+ _, resp, err := web.Get(web.DefaultSecurity, url)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("unexpected status (%s): %v", web.PasswordRedacted(url), resp.Status)
+ }
+ return resp.Body, nil
+}
+
func (p *proxyRepo) Versions(prefix string) ([]string, error) {
- var data []byte
- err := webGetBytes(p.url+"/@v/list", &data)
+ data, err := p.getBytes("@v/list")
if err != nil {
return nil, err
}
@@ -142,8 +198,7 @@
}
func (p *proxyRepo) latest() (*RevInfo, error) {
- var data []byte
- err := webGetBytes(p.url+"/@v/list", &data)
+ data, err := p.getBytes("@v/list")
if err != nil {
return nil, err
}
@@ -172,12 +227,11 @@
}
func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
- var data []byte
encRev, err := module.EncodeVersion(rev)
if err != nil {
return nil, err
}
- err = webGetBytes(p.url+"/@v/"+pathEscape(encRev)+".info", &data)
+ data, err := p.getBytes("@v/" + encRev + ".info")
if err != nil {
return nil, err
}
@@ -189,9 +243,7 @@
}
func (p *proxyRepo) Latest() (*RevInfo, error) {
- var data []byte
- u := p.url + "/@latest"
- err := webGetBytes(u, &data)
+ data, err := p.getBytes("@latest")
if err != nil {
// TODO return err if not 404
return p.latest()
@@ -204,12 +256,11 @@
}
func (p *proxyRepo) GoMod(version string) ([]byte, error) {
- var data []byte
encVer, err := module.EncodeVersion(version)
if err != nil {
return nil, err
}
- err = webGetBytes(p.url+"/@v/"+pathEscape(encVer)+".mod", &data)
+ data, err := p.getBytes("@v/" + encVer + ".mod")
if err != nil {
return nil, err
}
@@ -217,12 +268,11 @@
}
func (p *proxyRepo) Zip(dst io.Writer, version string) error {
- var body io.ReadCloser
encVer, err := module.EncodeVersion(version)
if err != nil {
return err
}
- err = webGetBody(p.url+"/@v/"+pathEscape(encVer)+".zip", &body)
+ body, err := p.getBody("@v/" + encVer + ".zip")
if err != nil {
return err
}
@@ -242,5 +292,5 @@
// That is, it escapes things like ? and # (which really shouldn't appear anyway).
// It does not escape / to %2F: our REST API is designed so that / can be left as is.
func pathEscape(s string) string {
- return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
+ return strings.ReplaceAll(urlpkg.PathEscape(s), "%2F", "/")
}
diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go
index ab6e46d..c3c4ade 100644
--- a/src/cmd/go/internal/modfetch/repo.go
+++ b/src/cmd/go/internal/modfetch/repo.go
@@ -209,7 +209,7 @@
return lookupProxy(path)
}
- security := web.Secure
+ security := web.SecureOnly
if get.Insecure {
security = web.Insecure
}
@@ -254,7 +254,7 @@
// Note: Because we are converting a code reference from a legacy
// version control system, we ignore meta tags about modules
// and use only direct source control entries (get.IgnoreMod).
- security := web.Secure
+ security := web.SecureOnly
if get.Insecure {
security = web.Insecure
}
diff --git a/src/cmd/go/internal/modfetch/web.go b/src/cmd/go/internal/modfetch/web.go
deleted file mode 100644
index b327bf2..0000000
--- a/src/cmd/go/internal/modfetch/web.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !cmd_go_bootstrap
-
-package modfetch
-
-import (
- "io"
-
- web "cmd/go/internal/web2"
-)
-
-// webGetGoGet fetches a go-get=1 URL and returns the body in *body.
-// It allows non-200 responses, as usual for these URLs.
-func webGetGoGet(url string, body *io.ReadCloser) error {
- return web.Get(url, web.Non200OK(), web.Body(body))
-}
-
-// webGetBytes returns the body returned by an HTTP GET, as a []byte.
-// It insists on a 200 response.
-func webGetBytes(url string, body *[]byte) error {
- return web.Get(url, web.ReadAllBody(body))
-}
-
-// webGetBody returns the body returned by an HTTP GET, as a io.ReadCloser.
-// It insists on a 200 response.
-func webGetBody(url string, body *io.ReadCloser) error {
- return web.Get(url, web.Body(body))
-}
diff --git a/src/cmd/go/internal/web/api.go b/src/cmd/go/internal/web/api.go
new file mode 100644
index 0000000..5dc81de
--- /dev/null
+++ b/src/cmd/go/internal/web/api.go
@@ -0,0 +1,102 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package web defines minimal helper routines for accessing HTTP/HTTPS
+// resources without requiring external dependenicies on the net package.
+//
+// If the cmd_go_bootstrap build tag is present, web avoids the use of the net
+// package and returns errors for all network operations.
+package web
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ urlpkg "net/url"
+)
+
+// SecurityMode specifies whether a function should make network
+// calls using insecure transports (eg, plain text HTTP).
+// The zero value is "secure".
+type SecurityMode int
+
+const (
+ SecureOnly SecurityMode = iota // Reject plain HTTP; validate HTTPS.
+ DefaultSecurity // Allow plain HTTP if explicit; validate HTTPS.
+ Insecure // Allow plain HTTP if not explicitly HTTPS; skip HTTPS validation.
+)
+
+type HTTPError struct {
+ status string
+ StatusCode int
+ url *urlpkg.URL
+}
+
+func (e *HTTPError) Error() string {
+ return fmt.Sprintf("%s: %s", e.url, e.status)
+}
+
+// GetBytes returns the body of the requested resource, or an error if the
+// response status was not http.StatusOk.
+//
+// GetBytes is a convenience wrapper around Get.
+func GetBytes(url *urlpkg.URL) ([]byte, error) {
+ url, resp, err := Get(DefaultSecurity, url)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url}
+ return nil, err
+ }
+ b, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("%s: %v", url, err)
+ }
+ return b, nil
+}
+
+type Response struct {
+ Status string
+ StatusCode int
+ Header map[string][]string
+ Body io.ReadCloser
+}
+
+// Get returns the body of the HTTP or HTTPS resource specified at the given URL.
+//
+// If the URL does not include an explicit scheme, Get first tries "https".
+// If the server does not respond under that scheme and the security mode is
+// Insecure, Get then tries "http".
+// The returned URL indicates which scheme was actually used.
+//
+// For the "https" scheme only, credentials are attached using the
+// cmd/go/internal/auth package. If the URL itself includes a username and
+// password, it will not be attempted under the "http" scheme unless the
+// security mode is Insecure.
+//
+// Get returns a non-nil error only if the request did not receive a response
+// under any applicable scheme. (A non-2xx response does not cause an error.)
+func Get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
+ return get(security, url)
+}
+
+// PasswordRedacted returns url directly if it does not encode a password,
+// or else a copy of url with the password redacted.
+func PasswordRedacted(url *urlpkg.URL) *urlpkg.URL {
+ if url.User != nil {
+ if _, ok := url.User.Password(); ok {
+ redacted := *url
+ redacted.User = urlpkg.UserPassword(url.User.Username(), "[redacted]")
+ return &redacted
+ }
+ }
+ return url
+}
+
+// OpenBrowser attempts to open the requested URL in a web browser.
+func OpenBrowser(url string) (opened bool) {
+ return openBrowser(url)
+}
diff --git a/src/cmd/go/internal/web/bootstrap.go b/src/cmd/go/internal/web/bootstrap.go
index d1d4621..84e9d35 100644
--- a/src/cmd/go/internal/web/bootstrap.go
+++ b/src/cmd/go/internal/web/bootstrap.go
@@ -6,32 +6,18 @@
// This code is compiled only into the bootstrap 'go' binary.
// These stubs avoid importing packages with large dependency
-// trees, like the use of "net/http" in vcs.go.
+// trees that potentially require C linking,
+// like the use of "net/http" in vcs.go.
package web
import (
"errors"
- "io"
+ urlpkg "net/url"
)
-var errHTTP = errors.New("no http in bootstrap go command")
-
-type HTTPError struct {
- StatusCode int
+func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
+ return nil, nil, errors.New("no http in bootstrap go command")
}
-func (e *HTTPError) Error() string {
- panic("unreachable")
-}
-
-func Get(url string) ([]byte, error) {
- return nil, errHTTP
-}
-
-func GetMaybeInsecure(importPath string, security SecurityMode) (string, io.ReadCloser, error) {
- return "", nil, errHTTP
-}
-
-func QueryEscape(s string) string { panic("unreachable") }
-func OpenBrowser(url string) bool { panic("unreachable") }
+func openBrowser(url string) bool { return false }
diff --git a/src/cmd/go/internal/web/http.go b/src/cmd/go/internal/web/http.go
index c1714b4d..0711f81 100644
--- a/src/cmd/go/internal/web/http.go
+++ b/src/cmd/go/internal/web/http.go
@@ -14,13 +14,12 @@
import (
"crypto/tls"
"fmt"
- "io"
- "io/ioutil"
"log"
"net/http"
- "net/url"
+ urlpkg "net/url"
"time"
+ "cmd/go/internal/auth"
"cmd/go/internal/cfg"
"cmd/internal/browser"
)
@@ -50,81 +49,92 @@
},
}
-type HTTPError struct {
- status string
- StatusCode int
- url string
-}
+func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
+ fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) {
+ if cfg.BuildV {
+ log.Printf("Fetching %s", url)
+ }
-func (e *HTTPError) Error() string {
- return fmt.Sprintf("%s: %s", e.url, e.status)
-}
-
-// Get returns the data from an HTTP GET request for the given URL.
-func Get(url string) ([]byte, error) {
- resp, err := securityPreservingHTTPClient.Get(url)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url}
-
- return nil, err
- }
- b, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("%s: %v", url, err)
- }
- return b, nil
-}
-
-// GetMaybeInsecure returns the body of either the importPath's
-// https resource or, if unavailable and permitted by the security mode, the http resource.
-func GetMaybeInsecure(importPath string, security SecurityMode) (urlStr string, body io.ReadCloser, err error) {
- fetch := func(scheme string) (urlStr string, res *http.Response, err error) {
- u, err := url.Parse(scheme + "://" + importPath)
+ req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
- return "", nil, err
+ return nil, nil, err
}
- u.RawQuery = "go-get=1"
- urlStr = u.String()
- if cfg.BuildV {
- log.Printf("Fetching %s", urlStr)
+ if url.Scheme == "https" {
+ auth.AddCredentials(req)
}
- if security == Insecure && scheme == "https" { // fail earlier
- res, err = impatientInsecureHTTPClient.Get(urlStr)
+
+ var res *http.Response
+ if security == Insecure && url.Scheme == "https" { // fail earlier
+ res, err = impatientInsecureHTTPClient.Do(req)
} else {
- res, err = securityPreservingHTTPClient.Get(urlStr)
+ res, err = securityPreservingHTTPClient.Do(req)
}
- return
+ return url, res, err
}
- closeBody := func(res *http.Response) {
- if res != nil {
- res.Body.Close()
+
+ var (
+ fetched *urlpkg.URL
+ res *http.Response
+ err error
+ )
+ if url.Scheme == "" || url.Scheme == "https" {
+ secure := new(urlpkg.URL)
+ *secure = *url
+ secure.Scheme = "https"
+
+ fetched, res, err = fetch(secure)
+ if err != nil {
+ if cfg.BuildV {
+ log.Printf("https fetch failed: %v", err)
+ }
+ if security != Insecure || url.Scheme == "https" {
+ // HTTPS failed, and we can't fall back to plain HTTP.
+ // Report the error from the HTTPS attempt.
+ return nil, nil, err
+ }
}
}
- urlStr, res, err := fetch("https")
- if err != nil {
- if cfg.BuildV {
- log.Printf("https fetch failed: %v", err)
+
+ if res == nil {
+ switch url.Scheme {
+ case "http":
+ if security == SecureOnly {
+ return nil, nil, fmt.Errorf("URL %q is not secure", PasswordRedacted(url))
+ }
+ case "":
+ if security != Insecure {
+ panic("should have returned after HTTPS failure")
+ }
+ default:
+ return nil, nil, fmt.Errorf("unsupported scheme %s", url.Scheme)
}
- if security == Insecure {
- closeBody(res)
- urlStr, res, err = fetch("http")
+
+ insecure := new(urlpkg.URL)
+ *insecure = *url
+ insecure.Scheme = "http"
+ if insecure.User != nil && security != Insecure {
+ return nil, nil, fmt.Errorf("refusing to pass credentials to insecure URL %q", PasswordRedacted(insecure))
+ }
+
+ fetched, res, err = fetch(insecure)
+ if err != nil {
+ // HTTP failed, and we already tried HTTPS if applicable.
+ // Report the error from the HTTP attempt.
+ return nil, nil, err
}
}
- if err != nil {
- closeBody(res)
- return "", nil, err
- }
+
// Note: accepting a non-200 OK here, so people can serve a
// meta import in their http 404 page.
if cfg.BuildV {
- log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode)
+ log.Printf("Parsing meta tags from %s (status code %d)", PasswordRedacted(fetched), res.StatusCode)
}
- return urlStr, res.Body, nil
+ return fetched, &Response{
+ Status: res.Status,
+ StatusCode: res.StatusCode,
+ Header: map[string][]string(res.Header),
+ Body: res.Body,
+ }, nil
}
-func QueryEscape(s string) string { return url.QueryEscape(s) }
-func OpenBrowser(url string) bool { return browser.Open(url) }
+func openBrowser(url string) bool { return browser.Open(url) }
diff --git a/src/cmd/go/internal/web/security.go b/src/cmd/go/internal/web/security.go
deleted file mode 100644
index 1dc6f1b..0000000
--- a/src/cmd/go/internal/web/security.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package web defines helper routines for accessing HTTP/HTTPS resources.
-package web
-
-// SecurityMode specifies whether a function should make network
-// calls using insecure transports (eg, plain text HTTP).
-// The zero value is "secure".
-type SecurityMode int
-
-const (
- Secure SecurityMode = iota
- Insecure
-)
diff --git a/src/cmd/go/internal/web2/web.go b/src/cmd/go/internal/web2/web.go
deleted file mode 100644
index 02b828f..0000000
--- a/src/cmd/go/internal/web2/web.go
+++ /dev/null
@@ -1,345 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package web2
-
-import (
- "bytes"
- "cmd/go/internal/base"
- "cmd/go/internal/cfg"
- "encoding/json"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "runtime"
- "runtime/debug"
- "strings"
- "sync"
-)
-
-var TraceGET = false
-var webstack = false
-
-func init() {
- flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests")
- flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests")
-}
-
-type netrcLine struct {
- machine string
- login string
- password string
-}
-
-var (
- netrcOnce sync.Once
- netrc []netrcLine
- netrcErr error
-)
-
-func parseNetrc(data string) []netrcLine {
- // See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
- // for documentation on the .netrc format.
- var nrc []netrcLine
- var l netrcLine
- inMacro := false
- for _, line := range strings.Split(data, "\n") {
- if inMacro {
- if line == "" {
- inMacro = false
- }
- continue
- }
-
- f := strings.Fields(line)
- i := 0
- for ; i < len(f)-1; i += 2 {
- // Reset at each "machine" token.
- // “The auto-login process searches the .netrc file for a machine token
- // that matches […]. Once a match is made, the subsequent .netrc tokens
- // are processed, stopping when the end of file is reached or another
- // machine or a default token is encountered.”
- switch f[i] {
- case "machine":
- l = netrcLine{machine: f[i+1]}
- case "default":
- break
- case "login":
- l.login = f[i+1]
- case "password":
- l.password = f[i+1]
- case "macdef":
- // “A macro is defined with the specified name; its contents begin with
- // the next .netrc line and continue until a null line (consecutive
- // new-line characters) is encountered.”
- inMacro = true
- }
- if l.machine != "" && l.login != "" && l.password != "" {
- nrc = append(nrc, l)
- l = netrcLine{}
- }
- }
-
- if i < len(f) && f[i] == "default" {
- // “There can be only one default token, and it must be after all machine tokens.”
- break
- }
- }
-
- return nrc
-}
-
-func havePassword(machine string) bool {
- netrcOnce.Do(readNetrc)
- for _, line := range netrc {
- if line.machine == machine {
- return true
- }
- }
- return false
-}
-
-func netrcPath() (string, error) {
- if env := os.Getenv("NETRC"); env != "" {
- return env, nil
- }
- dir, err := os.UserHomeDir()
- if err != nil {
- return "", err
- }
- base := ".netrc"
- if runtime.GOOS == "windows" {
- base = "_netrc"
- }
- return filepath.Join(dir, base), nil
-}
-
-func readNetrc() {
- path, err := netrcPath()
- if err != nil {
- netrcErr = err
- return
- }
-
- data, err := ioutil.ReadFile(path)
- if err != nil {
- if !os.IsNotExist(err) {
- netrcErr = err
- }
- return
- }
-
- netrc = parseNetrc(string(data))
-}
-
-type getState struct {
- req *http.Request
- resp *http.Response
- body io.ReadCloser
- non200ok bool
-}
-
-type Option interface {
- option(*getState) error
-}
-
-func Non200OK() Option {
- return optionFunc(func(g *getState) error {
- g.non200ok = true
- return nil
- })
-}
-
-type optionFunc func(*getState) error
-
-func (f optionFunc) option(g *getState) error {
- return f(g)
-}
-
-func DecodeJSON(dst interface{}) Option {
- return optionFunc(func(g *getState) error {
- if g.resp != nil {
- return json.NewDecoder(g.body).Decode(dst)
- }
- return nil
- })
-}
-
-func ReadAllBody(body *[]byte) Option {
- return optionFunc(func(g *getState) error {
- if g.resp != nil {
- var err error
- *body, err = ioutil.ReadAll(g.body)
- return err
- }
- return nil
- })
-}
-
-func Body(body *io.ReadCloser) Option {
- return optionFunc(func(g *getState) error {
- if g.resp != nil {
- *body = g.body
- g.body = nil
- }
- return nil
- })
-}
-
-func Header(hdr *http.Header) Option {
- return optionFunc(func(g *getState) error {
- if g.resp != nil {
- *hdr = CopyHeader(g.resp.Header)
- }
- return nil
- })
-}
-
-func CopyHeader(hdr http.Header) http.Header {
- if hdr == nil {
- return nil
- }
- h2 := make(http.Header)
- for k, v := range hdr {
- v2 := make([]string, len(v))
- copy(v2, v)
- h2[k] = v2
- }
- return h2
-}
-
-var cache struct {
- mu sync.Mutex
- byURL map[string]*cacheEntry
-}
-
-type cacheEntry struct {
- mu sync.Mutex
- resp *http.Response
- body []byte
-}
-
-var httpDo = http.DefaultClient.Do
-
-func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) {
- if do == nil {
- do = http.DefaultClient.Do
- }
- httpDo = do
-}
-
-func Get(url string, options ...Option) error {
- if TraceGET || webstack || cfg.BuildV {
- log.Printf("Fetching %s", url)
- if webstack {
- log.Println(string(debug.Stack()))
- }
- }
-
- req, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return err
- }
-
- netrcOnce.Do(readNetrc)
- for _, l := range netrc {
- if l.machine == req.URL.Host {
- req.SetBasicAuth(l.login, l.password)
- break
- }
- }
-
- g := &getState{req: req}
- for _, o := range options {
- if err := o.option(g); err != nil {
- return err
- }
- }
-
- cache.mu.Lock()
- e := cache.byURL[url]
- if e == nil {
- e = new(cacheEntry)
- if !strings.HasPrefix(url, "file:") {
- if cache.byURL == nil {
- cache.byURL = make(map[string]*cacheEntry)
- }
- cache.byURL[url] = e
- }
- }
- cache.mu.Unlock()
-
- e.mu.Lock()
- if strings.HasPrefix(url, "file:") {
- body, err := ioutil.ReadFile(req.URL.Path)
- if err != nil {
- e.mu.Unlock()
- return err
- }
- e.body = body
- e.resp = &http.Response{
- StatusCode: 200,
- }
- } else if e.resp == nil {
- resp, err := httpDo(req)
- if err != nil {
- e.mu.Unlock()
- return err
- }
- e.resp = resp
- // TODO: Spool to temp file.
- body, err := ioutil.ReadAll(resp.Body)
- resp.Body.Close()
- resp.Body = nil
- if err != nil {
- e.mu.Unlock()
- return err
- }
- e.body = body
- }
- g.resp = e.resp
- g.body = ioutil.NopCloser(bytes.NewReader(e.body))
- e.mu.Unlock()
-
- defer func() {
- if g.body != nil {
- g.body.Close()
- }
- }()
-
- if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") {
- base.Errorf("%s", githubMessage)
- }
- if !g.non200ok && g.resp.StatusCode != 200 {
- return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status)
- }
-
- for _, o := range options {
- if err := o.option(g); err != nil {
- return err
- }
- }
- return err
-}
-
-var githubMessage = `go: 403 response from api.github.com
-
-GitHub applies fairly small rate limits to unauthenticated users, and
-you appear to be hitting them. To authenticate, please visit
-https://github.com/settings/tokens and click "Generate New Token" to
-create a Personal Access Token. The token only needs "public_repo"
-scope, but you can add "repo" if you want to access private
-repositories too.
-
-Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows):
-
- machine api.github.com login YOU password TOKEN
-
-Sorry for the interruption.
-`
diff --git a/src/cmd/go/internal/webtest/test.go b/src/cmd/go/internal/webtest/test.go
deleted file mode 100644
index 94b20a3..0000000
--- a/src/cmd/go/internal/webtest/test.go
+++ /dev/null
@@ -1,314 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package webtest
-
-import (
- "bufio"
- "bytes"
- "encoding/hex"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net/http"
- "os"
- "sort"
- "strconv"
- "strings"
- "sync"
- "unicode/utf8"
-
- web "cmd/go/internal/web2"
-)
-
-var mode = flag.String("webtest", "replay", "set webtest `mode` - record, replay, bypass")
-
-func Hook() {
- if *mode == "bypass" {
- return
- }
- web.SetHTTPDoForTesting(Do)
-}
-
-func Unhook() {
- web.SetHTTPDoForTesting(nil)
-}
-
-func Print() {
- web.SetHTTPDoForTesting(DoPrint)
-}
-
-var responses struct {
- mu sync.Mutex
- byURL map[string]*respEntry
-}
-
-type respEntry struct {
- status string
- code int
- hdr http.Header
- body []byte
-}
-
-func Serve(url string, status string, hdr http.Header, body []byte) {
- if status == "" {
- status = "200 OK"
- }
- code, err := strconv.Atoi(strings.Fields(status)[0])
- if err != nil {
- panic("bad Serve status - " + status + " - " + err.Error())
- }
-
- responses.mu.Lock()
- defer responses.mu.Unlock()
-
- if responses.byURL == nil {
- responses.byURL = make(map[string]*respEntry)
- }
- responses.byURL[url] = &respEntry{status: status, code: code, hdr: web.CopyHeader(hdr), body: body}
-}
-
-func Do(req *http.Request) (*http.Response, error) {
- if req.Method != "GET" {
- return nil, fmt.Errorf("bad method - must be GET")
- }
-
- responses.mu.Lock()
- e := responses.byURL[req.URL.String()]
- responses.mu.Unlock()
-
- if e == nil {
- if *mode == "record" {
- loaded.mu.Lock()
- if len(loaded.did) != 1 {
- loaded.mu.Unlock()
- return nil, fmt.Errorf("cannot use -webtest=record with multiple loaded response files")
- }
- var file string
- for file = range loaded.did {
- break
- }
- loaded.mu.Unlock()
- return doSave(file, req)
- }
- e = &respEntry{code: 599, status: "599 unexpected request (no canned response)"}
- }
- resp := &http.Response{
- Status: e.status,
- StatusCode: e.code,
- Header: web.CopyHeader(e.hdr),
- Body: ioutil.NopCloser(bytes.NewReader(e.body)),
- }
- return resp, nil
-}
-
-func DoPrint(req *http.Request) (*http.Response, error) {
- return doSave("", req)
-}
-
-func doSave(file string, req *http.Request) (*http.Response, error) {
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return nil, err
- }
- data, err := ioutil.ReadAll(resp.Body)
- resp.Body.Close()
- if err != nil {
- return nil, err
- }
- resp.Body = ioutil.NopCloser(bytes.NewReader(data))
-
- var f *os.File
- if file == "" {
- f = os.Stderr
- } else {
- f, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
- if err != nil {
- log.Fatal(err)
- }
- defer f.Close()
- }
-
- fmt.Fprintf(f, "GET %s\n", req.URL.String())
- fmt.Fprintf(f, "%s\n", resp.Status)
- var keys []string
- for k := range resp.Header {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- for _, k := range keys {
- if k == "Set-Cookie" {
- continue
- }
- for _, v := range resp.Header[k] {
- fmt.Fprintf(f, "%s: %s\n", k, v)
- }
- }
- fmt.Fprintf(f, "\n")
- if utf8.Valid(data) && !bytes.Contains(data, []byte("\nGET")) && !isHexDump(data) {
- fmt.Fprintf(f, "%s\n\n", data)
- } else {
- fmt.Fprintf(f, "%s\n", hex.Dump(data))
- }
- return resp, err
-}
-
-var loaded struct {
- mu sync.Mutex
- did map[string]bool
-}
-
-func LoadOnce(file string) {
- loaded.mu.Lock()
- if loaded.did[file] {
- loaded.mu.Unlock()
- return
- }
- if loaded.did == nil {
- loaded.did = make(map[string]bool)
- }
- loaded.did[file] = true
- loaded.mu.Unlock()
-
- f, err := os.Open(file)
- if err != nil {
- log.Fatal(err)
- }
- defer f.Close()
-
- b := bufio.NewReader(f)
- var ungetLine string
- nextLine := func() string {
- if ungetLine != "" {
- l := ungetLine
- ungetLine = ""
- return l
- }
- line, err := b.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- return ""
- }
- log.Fatalf("%s: unexpected read error: %v", file, err)
- }
- return line
- }
-
- for {
- line := nextLine()
- if line == "" { // EOF
- break
- }
- line = strings.TrimSpace(line)
- if strings.HasPrefix(line, "#") || line == "" {
- continue
- }
- if !strings.HasPrefix(line, "GET ") {
- log.Fatalf("%s: malformed GET line: %s", file, line)
- }
- url := line[len("GET "):]
- status := nextLine()
- if _, err := strconv.Atoi(strings.Fields(status)[0]); err != nil {
- log.Fatalf("%s: malformed status line (after GET %s): %s", file, url, status)
- }
- hdr := make(http.Header)
- for {
- kv := strings.TrimSpace(nextLine())
- if kv == "" {
- break
- }
- i := strings.Index(kv, ":")
- if i < 0 {
- log.Fatalf("%s: malformed header line (after GET %s): %s", file, url, kv)
- }
- k, v := kv[:i], strings.TrimSpace(kv[i+1:])
- hdr[k] = append(hdr[k], v)
- }
-
- var body []byte
- Body:
- for n := 0; ; n++ {
- line := nextLine()
- if n == 0 && isHexDump([]byte(line)) {
- ungetLine = line
- b, err := parseHexDump(nextLine)
- if err != nil {
- log.Fatalf("%s: malformed hex dump (after GET %s): %v", file, url, err)
- }
- body = b
- break
- }
- if line == "" { // EOF
- for i := 0; i < 2; i++ {
- if len(body) > 0 && body[len(body)-1] == '\n' {
- body = body[:len(body)-1]
- }
- }
- break
- }
- body = append(body, line...)
- for line == "\n" {
- line = nextLine()
- if strings.HasPrefix(line, "GET ") {
- ungetLine = line
- body = body[:len(body)-1]
- if len(body) > 0 {
- body = body[:len(body)-1]
- }
- break Body
- }
- body = append(body, line...)
- }
- }
-
- Serve(url, status, hdr, body)
- }
-}
-
-func isHexDump(data []byte) bool {
- return bytes.HasPrefix(data, []byte("00000000 ")) || bytes.HasPrefix(data, []byte("0000000 "))
-}
-
-// parseHexDump parses the hex dump in text, which should be the
-// output of "hexdump -C" or Plan 9's "xd -b" or Go's hex.Dump
-// and returns the original data used to produce the dump.
-// It is meant to enable storing golden binary files as text, so that
-// changes to the golden files can be seen during code reviews.
-func parseHexDump(nextLine func() string) ([]byte, error) {
- var out []byte
- for {
- line := nextLine()
- if line == "" || line == "\n" {
- break
- }
- if i := strings.Index(line, "|"); i >= 0 { // remove text dump
- line = line[:i]
- }
- f := strings.Fields(line)
- if len(f) > 1+16 {
- return nil, fmt.Errorf("parsing hex dump: too many fields on line %q", line)
- }
- if len(f) == 0 || len(f) == 1 && f[0] == "*" { // all zeros block omitted
- continue
- }
- addr64, err := strconv.ParseUint(f[0], 16, 0)
- if err != nil {
- return nil, fmt.Errorf("parsing hex dump: invalid address %q", f[0])
- }
- addr := int(addr64)
- if len(out) < addr {
- out = append(out, make([]byte, addr-len(out))...)
- }
- for _, x := range f[1:] {
- val, err := strconv.ParseUint(x, 16, 8)
- if err != nil {
- return nil, fmt.Errorf("parsing hexdump: invalid hex byte %q", x)
- }
- out = append(out, byte(val))
- }
- }
- return out, nil
-}
diff --git a/src/cmd/go/testdata/script/get_404_meta.txt b/src/cmd/go/testdata/script/get_404_meta.txt
new file mode 100644
index 0000000..32f13c9
--- /dev/null
+++ b/src/cmd/go/testdata/script/get_404_meta.txt
@@ -0,0 +1,10 @@
+# golang.org/issue/13037: 'go get' was not parsing <meta> tags in 404 served over HTTPS.
+
+[!net] skip
+
+env GO111MODULE=off
+go get -d -insecure bazil.org/fuse/fs/fstestutil
+
+env GO111MODULE=on
+env GOPROXY=direct
+go get -d -insecure bazil.org/fuse/fs/fstestutil
diff --git a/src/cmd/go/testdata/script/get_insecure_redirect.txt b/src/cmd/go/testdata/script/get_insecure_redirect.txt
index c3520bf..e05ced6 100644
--- a/src/cmd/go/testdata/script/get_insecure_redirect.txt
+++ b/src/cmd/go/testdata/script/get_insecure_redirect.txt
@@ -1,4 +1,3 @@
-# golang.org/issue/13037: 'go get' was not parsing <meta> tags in 404 served over HTTPS.
# golang.org/issue/29591: 'go get' was following plain-HTTP redirects even without -insecure.
[!net] skip
diff --git a/src/cmd/go/testdata/script/mod_auth.txt b/src/cmd/go/testdata/script/mod_auth.txt
new file mode 100644
index 0000000..b47db9c
--- /dev/null
+++ b/src/cmd/go/testdata/script/mod_auth.txt
@@ -0,0 +1,31 @@
+[!net] skip
+
+env GO111MODULE=on
+env GOPROXY=direct
+
+# Without credentials, downloading a module from a path that requires HTTPS
+# basic auth should fail.
+env NETRC=$WORK/empty
+! go list all
+
+# With credentials from a netrc file, it should succeed.
+env NETRC=$WORK/netrc
+go mod tidy
+go list all
+stdout vcs-test.golang.org/auth/or401
+stdout vcs-test.golang.org/auth/or404
+
+-- go.mod --
+module private.example.com
+-- main.go --
+package useprivate
+
+import (
+ _ "vcs-test.golang.org/auth/or401"
+ _ "vcs-test.golang.org/auth/or404"
+)
+-- $WORK/empty --
+-- $WORK/netrc --
+machine vcs-test.golang.org
+ login aladdin
+ password opensesame
