src/cmd/go/internal/web2/web.go - go - Git at Google

 // Copyright 2018 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package web2

 import (
 	"bytes"
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strings"
 	"sync"
 )

 var TraceGET = false
 var webstack = false

 func init() {
 	flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests")
 	flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests")
 }

 type netrcLine struct {
 	machine  string
 	login    string
 	password string
 }

 var (
 	netrcOnce sync.Once
 	netrc     []netrcLine
 	netrcErr  error
 )

 func parseNetrc(data string) []netrcLine {
 	// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
 	// for documentation on the .netrc format.
 	var nrc []netrcLine
 	var l netrcLine
 	inMacro := false
 	for _, line := range strings.Split(data, "\n") {
 		if inMacro {
 			if line == "" {
 				inMacro = false
 			}
 			continue
 		}

 		f := strings.Fields(line)
 		i := 0
 		for ; i < len(f)-1; i += 2 {
 			// Reset at each "machine" token.
 			// “The auto-login process searches the .netrc file for a machine token
 			// that matches […]. Once a match is made, the subsequent .netrc tokens
 			// are processed, stopping when the end of file is reached or another
 			// machine or a default token is encountered.”
 			switch f[i] {
 			case "machine":
 				l = netrcLine{machine: f[i+1]}
 			case "default":
 				break
 			case "login":
 				l.login = f[i+1]
 			case "password":
 				l.password = f[i+1]
 			case "macdef":
 				// “A macro is defined with the specified name; its contents begin with
 				// the next .netrc line and continue until a null line (consecutive
 				// new-line characters) is encountered.”
 				inMacro = true
 			}
 			if l.machine != "" && l.login != "" && l.password != "" {
 				nrc = append(nrc, l)
 				l = netrcLine{}
 			}
 		}

 		if i < len(f) && f[i] == "default" {
 			// “There can be only one default token, and it must be after all machine tokens.”
 			break
 		}
 	}

 	return nrc
 }

 func havePassword(machine string) bool {
 	netrcOnce.Do(readNetrc)
 	for _, line := range netrc {
 		if line.machine == machine {
 			return true
 		}
 	}
 	return false
 }

 func netrcPath() (string, error) {
 	if env := os.Getenv("NETRC"); env != "" {
 		return env, nil
 	}
 	dir, err := os.UserHomeDir()
 	if err != nil {
 		return "", err
 	}
 	base := ".netrc"
 	if runtime.GOOS == "windows" {
 		base = "_netrc"
 	}
 	return filepath.Join(dir, base), nil
 }

 func readNetrc() {
 	path, err := netrcPath()
 	if err != nil {
 		netrcErr = err
 		return
 	}

 	data, err := ioutil.ReadFile(path)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			netrcErr = err
 		}
 		return
 	}

 	netrc = parseNetrc(string(data))
 }

 type getState struct {
 	req      *http.Request
 	resp     *http.Response
 	body     io.ReadCloser
 	non200ok bool
 }

 type Option interface {
 	option(*getState) error
 }

 func Non200OK() Option {
 	return optionFunc(func(g *getState) error {
 		g.non200ok = true
 		return nil
 	})
 }

 type optionFunc func(*getState) error

 func (f optionFunc) option(g *getState) error {
 	return f(g)
 }

 func DecodeJSON(dst interface{}) Option {
 	return optionFunc(func(g *getState) error {
 		if g.resp != nil {
 			return json.NewDecoder(g.body).Decode(dst)
 		}
 		return nil
 	})
 }

 func ReadAllBody(body *[]byte) Option {
 	return optionFunc(func(g *getState) error {
 		if g.resp != nil {
 			var err error
 			*body, err = ioutil.ReadAll(g.body)
 			return err
 		}
 		return nil
 	})
 }

 func Body(body *io.ReadCloser) Option {
 	return optionFunc(func(g *getState) error {
 		if g.resp != nil {
 			*body = g.body
 			g.body = nil
 		}
 		return nil
 	})
 }

 func Header(hdr *http.Header) Option {
 	return optionFunc(func(g *getState) error {
 		if g.resp != nil {
 			*hdr = CopyHeader(g.resp.Header)
 		}
 		return nil
 	})
 }

 func CopyHeader(hdr http.Header) http.Header {
 	if hdr == nil {
 		return nil
 	}
 	h2 := make(http.Header)
 	for k, v := range hdr {
 		v2 := make([]string, len(v))
 		copy(v2, v)
 		h2[k] = v2
 	}
 	return h2
 }

 var cache struct {
 	mu    sync.Mutex
 	byURL map[string]*cacheEntry
 }

 type cacheEntry struct {
 	mu   sync.Mutex
 	resp *http.Response
 	body []byte
 }

 var httpDo = http.DefaultClient.Do

 func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) {
 	if do == nil {
 		do = http.DefaultClient.Do
 	}
 	httpDo = do
 }

 func Get(url string, options ...Option) error {
 	if TraceGET || webstack || cfg.BuildV {
 		log.Printf("Fetching %s", url)
 		if webstack {
 			log.Println(string(debug.Stack()))
 		}
 	}

 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return err
 	}

 	netrcOnce.Do(readNetrc)
 	for _, l := range netrc {
 		if l.machine == req.URL.Host {
 			req.SetBasicAuth(l.login, l.password)
 			break
 		}
 	}

 	g := &getState{req: req}
 	for _, o := range options {
 		if err := o.option(g); err != nil {
 			return err
 		}
 	}

 	cache.mu.Lock()
 	e := cache.byURL[url]
 	if e == nil {
 		e = new(cacheEntry)
 		if !strings.HasPrefix(url, "file:") {
 			if cache.byURL == nil {
 				cache.byURL = make(map[string]*cacheEntry)
 			}
 			cache.byURL[url] = e
 		}
 	}
 	cache.mu.Unlock()

 	e.mu.Lock()
 	if strings.HasPrefix(url, "file:") {
 		body, err := ioutil.ReadFile(req.URL.Path)
 		if err != nil {
 			e.mu.Unlock()
 			return err
 		}
 		e.body = body
 		e.resp = &http.Response{
 			StatusCode: 200,
 		}
 	} else if e.resp == nil {
 		resp, err := httpDo(req)
 		if err != nil {
 			e.mu.Unlock()
 			return err
 		}
 		e.resp = resp
 		// TODO: Spool to temp file.
 		body, err := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
 		resp.Body = nil
 		if err != nil {
 			e.mu.Unlock()
 			return err
 		}
 		e.body = body
 	}
 	g.resp = e.resp
 	g.body = ioutil.NopCloser(bytes.NewReader(e.body))
 	e.mu.Unlock()

 	defer func() {
 		if g.body != nil {
 			g.body.Close()
 		}
 	}()

 	if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") {
 		base.Errorf("%s", githubMessage)
 	}
 	if !g.non200ok && g.resp.StatusCode != 200 {
 		return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status)
 	}

 	for _, o := range options {
 		if err := o.option(g); err != nil {
 			return err
 		}
 	}
 	return err
 }

 var githubMessage = `go: 403 response from api.github.com

 GitHub applies fairly small rate limits to unauthenticated users, and
 you appear to be hitting them. To authenticate, please visit
 https://github.com/settings/tokens and click "Generate New Token" to
 create a Personal Access Token. The token only needs "public_repo"
 scope, but you can add "repo" if you want to access private
 repositories too.

 Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows):

     machine api.github.com login YOU password TOKEN

 Sorry for the interruption.
 `
	// Copyright 2018 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package web2

	import (
	"bytes"
	"cmd/go/internal/base"
	"cmd/go/internal/cfg"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	)

	var TraceGET = false
	var webstack = false

	func init() {
	flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests")
	flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests")
	}

	type netrcLine struct {
	machine string
	login string
	password string
	}

	var (
	netrcOnce sync.Once
	netrc []netrcLine
	netrcErr error
	)

	func parseNetrc(data string) []netrcLine {
	// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
	// for documentation on the .netrc format.
	var nrc []netrcLine
	var l netrcLine
	inMacro := false
	for _, line := range strings.Split(data, "\n") {
	if inMacro {
	if line == "" {
	inMacro = false
	}
	continue
	}

	f := strings.Fields(line)
	i := 0
	for ; i < len(f)-1; i += 2 {
	// Reset at each "machine" token.
	// “The auto-login process searches the .netrc file for a machine token
	// that matches […]. Once a match is made, the subsequent .netrc tokens
	// are processed, stopping when the end of file is reached or another
	// machine or a default token is encountered.”
	switch f[i] {
	case "machine":
	l = netrcLine{machine: f[i+1]}
	case "default":
	break
	case "login":
	l.login = f[i+1]
	case "password":
	l.password = f[i+1]
	case "macdef":
	// “A macro is defined with the specified name; its contents begin with
	// the next .netrc line and continue until a null line (consecutive
	// new-line characters) is encountered.”
	inMacro = true
	}
	if l.machine != "" && l.login != "" && l.password != "" {
	nrc = append(nrc, l)
	l = netrcLine{}
	}
	}

	if i < len(f) && f[i] == "default" {
	// “There can be only one default token, and it must be after all machine tokens.”
	break
	}
	}

	return nrc
	}

	func havePassword(machine string) bool {
	netrcOnce.Do(readNetrc)
	for _, line := range netrc {
	if line.machine == machine {
	return true
	}
	}
	return false
	}

	func netrcPath() (string, error) {
	if env := os.Getenv("NETRC"); env != "" {
	return env, nil
	}
	dir, err := os.UserHomeDir()
	if err != nil {
	return "", err
	}
	base := ".netrc"
	if runtime.GOOS == "windows" {
	base = "_netrc"
	}
	return filepath.Join(dir, base), nil
	}

	func readNetrc() {
	path, err := netrcPath()
	if err != nil {
	netrcErr = err
	return
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
	if !os.IsNotExist(err) {
	netrcErr = err
	}
	return
	}

	netrc = parseNetrc(string(data))
	}

	type getState struct {
	req *http.Request
	resp *http.Response
	body io.ReadCloser
	non200ok bool
	}

	type Option interface {
	option(*getState) error
	}

	func Non200OK() Option {
	return optionFunc(func(g *getState) error {
	g.non200ok = true
	return nil
	})
	}

	type optionFunc func(*getState) error

	func (f optionFunc) option(g *getState) error {
	return f(g)
	}

	func DecodeJSON(dst interface{}) Option {
	return optionFunc(func(g *getState) error {
	if g.resp != nil {
	return json.NewDecoder(g.body).Decode(dst)
	}
	return nil
	})
	}

	func ReadAllBody(body *[]byte) Option {
	return optionFunc(func(g *getState) error {
	if g.resp != nil {
	var err error
	*body, err = ioutil.ReadAll(g.body)
	return err
	}
	return nil
	})
	}

	func Body(body *io.ReadCloser) Option {
	return optionFunc(func(g *getState) error {
	if g.resp != nil {
	*body = g.body
	g.body = nil
	}
	return nil
	})
	}

	func Header(hdr *http.Header) Option {
	return optionFunc(func(g *getState) error {
	if g.resp != nil {
	*hdr = CopyHeader(g.resp.Header)
	}
	return nil
	})
	}

	func CopyHeader(hdr http.Header) http.Header {
	if hdr == nil {
	return nil
	}
	h2 := make(http.Header)
	for k, v := range hdr {
	v2 := make([]string, len(v))
	copy(v2, v)
	h2[k] = v2
	}
	return h2
	}

	var cache struct {
	mu sync.Mutex
	byURL map[string]*cacheEntry
	}

	type cacheEntry struct {
	mu sync.Mutex
	resp *http.Response
	body []byte
	}

	var httpDo = http.DefaultClient.Do

	func SetHTTPDoForTesting(do func(http.Request) (http.Response, error)) {
	if do == nil {
	do = http.DefaultClient.Do
	}
	httpDo = do
	}

	func Get(url string, options ...Option) error {
	if TraceGET \|\| webstack \|\| cfg.BuildV {
	log.Printf("Fetching %s", url)
	if webstack {
	log.Println(string(debug.Stack()))
	}
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
	return err
	}

	netrcOnce.Do(readNetrc)
	for _, l := range netrc {
	if l.machine == req.URL.Host {
	req.SetBasicAuth(l.login, l.password)
	break
	}
	}

	g := &getState{req: req}
	for _, o := range options {
	if err := o.option(g); err != nil {
	return err
	}
	}

	cache.mu.Lock()
	e := cache.byURL[url]
	if e == nil {
	e = new(cacheEntry)
	if !strings.HasPrefix(url, "file:") {
	if cache.byURL == nil {
	cache.byURL = make(map[string]*cacheEntry)
	}
	cache.byURL[url] = e
	}
	}
	cache.mu.Unlock()

	e.mu.Lock()
	if strings.HasPrefix(url, "file:") {
	body, err := ioutil.ReadFile(req.URL.Path)
	if err != nil {
	e.mu.Unlock()
	return err
	}
	e.body = body
	e.resp = &http.Response{
	StatusCode: 200,
	}
	} else if e.resp == nil {
	resp, err := httpDo(req)
	if err != nil {
	e.mu.Unlock()
	return err
	}
	e.resp = resp
	// TODO: Spool to temp file.
	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	resp.Body = nil
	if err != nil {
	e.mu.Unlock()
	return err
	}
	e.body = body
	}
	g.resp = e.resp
	g.body = ioutil.NopCloser(bytes.NewReader(e.body))
	e.mu.Unlock()

	defer func() {
	if g.body != nil {
	g.body.Close()
	}
	}()

	if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") {
	base.Errorf("%s", githubMessage)
	}
	if !g.non200ok && g.resp.StatusCode != 200 {
	return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status)
	}

	for _, o := range options {
	if err := o.option(g); err != nil {
	return err
	}
	}
	return err
	}

	var githubMessage = `go: 403 response from api.github.com

	GitHub applies fairly small rate limits to unauthenticated users, and
	you appear to be hitting them. To authenticate, please visit
	https://github.com/settings/tokens and click "Generate New Token" to
	create a Personal Access Token. The token only needs "public_repo"
	scope, but you can add "repo" if you want to access private
	repositories too.

	Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows):

	machine api.github.com login YOU password TOKEN

	Sorry for the interruption.
	`