blob: 88d444eca2dea1b2e1be85e09ed213b0b935dcda [file] [log] [blame]
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// HTTP client. See RFC 2616.
//
// This is the high-level Client interface.
// The low-level implementation is in transport.go.
package http
import (
"encoding/base64"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
)
// A Client is an HTTP client. Its zero value (DefaultClient) is a
// usable client that uses DefaultTransport.
//
// The Client's Transport typically has internal state (cached TCP
// connections), so Clients should be reused instead of created as
// needed. Clients are safe for concurrent use by multiple goroutines.
//
// A Client is higher-level than a RoundTripper (such as Transport)
// and additionally handles HTTP details such as cookies and
// redirects.
type Client struct {
// Transport specifies the mechanism by which individual
// HTTP requests are made.
// If nil, DefaultTransport is used.
Transport RoundTripper
// CheckRedirect specifies the policy for handling redirects.
// If CheckRedirect is not nil, the client calls it before
// following an HTTP redirect. The arguments req and via are
// the upcoming request and the requests made already, oldest
// first. If CheckRedirect returns an error, the Client's Get
// method returns both the previous Response and
// CheckRedirect's error (wrapped in a url.Error) instead of
// issuing the Request req.
//
// If CheckRedirect is nil, the Client uses its default policy,
// which is to stop after 10 consecutive requests.
CheckRedirect func(req *Request, via []*Request) error
// Jar specifies the cookie jar.
// If Jar is nil, cookies are not sent in requests and ignored
// in responses.
Jar CookieJar
// Timeout specifies a time limit for requests made by this
// Client. The timeout includes connection time, any
// redirects, and reading the response body. The timer remains
// running after Get, Head, Post, or Do return and will
// interrupt reading of the Response.Body.
//
// A Timeout of zero means no timeout.
//
// The Client's Transport must support the CancelRequest
// method or Client will return errors when attempting to make
// a request with Get, Head, Post, or Do. Client's default
// Transport (DefaultTransport) supports CancelRequest.
Timeout time.Duration
}
// DefaultClient is the default Client and is used by Get, Head, and Post.
var DefaultClient = &Client{}
// RoundTripper is an interface representing the ability to execute a
// single HTTP transaction, obtaining the Response for a given Request.
//
// A RoundTripper must be safe for concurrent use by multiple
// goroutines.
type RoundTripper interface {
// RoundTrip executes a single HTTP transaction, returning
// the Response for the request req. RoundTrip should not
// attempt to interpret the response. In particular,
// RoundTrip must return err == nil if it obtained a response,
// regardless of the response's HTTP status code. A non-nil
// err should be reserved for failure to obtain a response.
// Similarly, RoundTrip should not attempt to handle
// higher-level protocol details such as redirects,
// authentication, or cookies.
//
// RoundTrip should not modify the request, except for
// consuming and closing the Body, including on errors. The
// request's URL and Header fields are guaranteed to be
// initialized.
RoundTrip(*Request) (*Response, error)
}
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
// refererForURL returns a referer without any authentication info or
// an empty string if lastReq scheme is https and newReq scheme is http.
func refererForURL(lastReq, newReq *url.URL) string {
// https://tools.ietf.org/html/rfc7231#section-5.5.2
// "Clients SHOULD NOT include a Referer header field in a
// (non-secure) HTTP request if the referring page was
// transferred with a secure protocol."
if lastReq.Scheme == "https" && newReq.Scheme == "http" {
return ""
}
referer := lastReq.String()
if lastReq.User != nil {
// This is not very efficient, but is the best we can
// do without:
// - introducing a new method on URL
// - creating a race condition
// - copying the URL struct manually, which would cause
// maintenance problems down the line
auth := lastReq.User.String() + "@"
referer = strings.Replace(referer, auth, "", 1)
}
return referer
}
// Used in Send to implement io.ReadCloser by bundling together the
// bufio.Reader through which we read the response, and the underlying
// network connection.
type readClose struct {
io.Reader
io.Closer
}
func (c *Client) send(req *Request) (*Response, error) {
if c.Jar != nil {
for _, cookie := range c.Jar.Cookies(req.URL) {
req.AddCookie(cookie)
}
}
resp, err := send(req, c.transport())
if err != nil {
return nil, err
}
if c.Jar != nil {
if rc := resp.Cookies(); len(rc) > 0 {
c.Jar.SetCookies(req.URL, rc)
}
}
return resp, err
}
// Do sends an HTTP request and returns an HTTP response, following
// policy (e.g. redirects, cookies, auth) as configured on the client.
//
// An error is returned if caused by client policy (such as
// CheckRedirect), or if there was an HTTP protocol error.
// A non-2xx response doesn't cause an error.
//
// When err is nil, resp always contains a non-nil resp.Body.
//
// Callers should close resp.Body when done reading from it. If
// resp.Body is not closed, the Client's underlying RoundTripper
// (typically Transport) may not be able to re-use a persistent TCP
// connection to the server for a subsequent "keep-alive" request.
//
// The request Body, if non-nil, will be closed by the underlying
// Transport, even on errors.
//
// Generally Get, Post, or PostForm will be used instead of Do.
func (c *Client) Do(req *Request) (resp *Response, err error) {
if req.Method == "GET" || req.Method == "HEAD" {
return c.doFollowingRedirects(req, shouldRedirectGet)
}
if req.Method == "POST" || req.Method == "PUT" {
return c.doFollowingRedirects(req, shouldRedirectPost)
}
return c.send(req)
}
func (c *Client) transport() RoundTripper {
if c.Transport != nil {
return c.Transport
}
return DefaultTransport
}
// send issues an HTTP request.
// Caller should close resp.Body when done reading from it.
func send(req *Request, t RoundTripper) (resp *Response, err error) {
if t == nil {
req.closeBody()
return nil, errors.New("http: no Client.Transport or DefaultTransport")
}
if req.URL == nil {
req.closeBody()
return nil, errors.New("http: nil Request.URL")
}
if req.RequestURI != "" {
req.closeBody()
return nil, errors.New("http: Request.RequestURI can't be set in client requests.")
}
// Most the callers of send (Get, Post, et al) don't need
// Headers, leaving it uninitialized. We guarantee to the
// Transport that this has been initialized, though.
if req.Header == nil {
req.Header = make(Header)
}
if u := req.URL.User; u != nil {
username := u.Username()
password, _ := u.Password()
req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
}
resp, err = t.RoundTrip(req)
if err != nil {
if resp != nil {
log.Printf("RoundTripper returned a response & error; ignoring response")
}
return nil, err
}
return resp, nil
}
// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt
// "To receive authorization, the client sends the userid and password,
// separated by a single colon (":") character, within a base64
// encoded string in the credentials."
// It is not meant to be urlencoded.
func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}
// True if the specified HTTP status code is one for which the Get utility should
// automatically redirect.
func shouldRedirectGet(statusCode int) bool {
switch statusCode {
case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
return true
}
return false
}
// True if the specified HTTP status code is one for which the Post utility should
// automatically redirect.
func shouldRedirectPost(statusCode int) bool {
switch statusCode {
case StatusFound, StatusSeeOther:
return true
}
return false
}
// Get issues a GET to the specified URL. If the response is one of the following
// redirect codes, Get follows the redirect, up to a maximum of 10 redirects:
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
//
// An error is returned if there were too many redirects or if there
// was an HTTP protocol error. A non-2xx response doesn't cause an
// error.
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
//
// Get is a wrapper around DefaultClient.Get.
func Get(url string) (resp *Response, err error) {
return DefaultClient.Get(url)
}
// Get issues a GET to the specified URL. If the response is one of the
// following redirect codes, Get follows the redirect after calling the
// Client's CheckRedirect function.
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
//
// An error is returned if the Client's CheckRedirect function fails
// or if there was an HTTP protocol error. A non-2xx response doesn't
// cause an error.
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
func (c *Client) Get(url string) (resp *Response, err error) {
req, err := NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return c.doFollowingRedirects(req, shouldRedirectGet)
}
func alwaysFalse() bool { return false }
func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) {
var base *url.URL
redirectChecker := c.CheckRedirect
if redirectChecker == nil {
redirectChecker = defaultCheckRedirect
}
var via []*Request
if ireq.URL == nil {
ireq.closeBody()
return nil, errors.New("http: nil Request.URL")
}
var reqmu sync.Mutex // guards req
req := ireq
var timer *time.Timer
var atomicWasCanceled int32 // atomic bool (1 or 0)
var wasCanceled = alwaysFalse
if c.Timeout > 0 {
wasCanceled = func() bool { return atomic.LoadInt32(&atomicWasCanceled) != 0 }
type canceler interface {
CancelRequest(*Request)
}
tr, ok := c.transport().(canceler)
if !ok {
return nil, fmt.Errorf("net/http: Client Transport of type %T doesn't support CancelRequest; Timeout not supported", c.transport())
}
timer = time.AfterFunc(c.Timeout, func() {
atomic.StoreInt32(&atomicWasCanceled, 1)
reqmu.Lock()
defer reqmu.Unlock()
tr.CancelRequest(req)
})
}
urlStr := "" // next relative or absolute URL to fetch (after first request)
redirectFailed := false
for redirect := 0; ; redirect++ {
if redirect != 0 {
nreq := new(Request)
nreq.Method = ireq.Method
if ireq.Method == "POST" || ireq.Method == "PUT" {
nreq.Method = "GET"
}
nreq.Header = make(Header)
nreq.URL, err = base.Parse(urlStr)
if err != nil {
break
}
if len(via) > 0 {
// Add the Referer header.
lastReq := via[len(via)-1]
if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" {
nreq.Header.Set("Referer", ref)
}
err = redirectChecker(nreq, via)
if err != nil {
redirectFailed = true
break
}
}
reqmu.Lock()
req = nreq
reqmu.Unlock()
}
urlStr = req.URL.String()
if resp, err = c.send(req); err != nil {
if wasCanceled() {
err = &httpError{
err: err.Error() + " (Client.Timeout exceeded while awaiting headers)",
timeout: true,
}
}
break
}
if shouldRedirect(resp.StatusCode) {
// Read the body if small so underlying TCP connection will be re-used.
// No need to check for errors: if it fails, Transport won't reuse it anyway.
const maxBodySlurpSize = 2 << 10
if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
}
resp.Body.Close()
if urlStr = resp.Header.Get("Location"); urlStr == "" {
err = fmt.Errorf("%d response missing Location header", resp.StatusCode)
break
}
base = req.URL
via = append(via, req)
continue
}
if timer != nil {
resp.Body = &cancelTimerBody{
t: timer,
rc: resp.Body,
reqWasCanceled: wasCanceled,
}
}
return resp, nil
}
method := ireq.Method
urlErr := &url.Error{
Op: method[0:1] + strings.ToLower(method[1:]),
URL: urlStr,
Err: err,
}
if redirectFailed {
// Special case for Go 1 compatibility: return both the response
// and an error if the CheckRedirect function failed.
// See http://golang.org/issue/3795
return resp, urlErr
}
if resp != nil {
resp.Body.Close()
}
return nil, urlErr
}
func defaultCheckRedirect(req *Request, via []*Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
return nil
}
// Post issues a POST to the specified URL.
//
// Caller should close resp.Body when done reading from it.
//
// Post is a wrapper around DefaultClient.Post
func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
return DefaultClient.Post(url, bodyType, body)
}
// Post issues a POST to the specified URL.
//
// Caller should close resp.Body when done reading from it.
//
// If the provided body is also an io.Closer, it is closed after the
// request.
func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
req, err := NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return c.doFollowingRedirects(req, shouldRedirectPost)
}
// PostForm issues a POST to the specified URL, with data's keys and
// values URL-encoded as the request body.
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
//
// PostForm is a wrapper around DefaultClient.PostForm
func PostForm(url string, data url.Values) (resp *Response, err error) {
return DefaultClient.PostForm(url, data)
}
// PostForm issues a POST to the specified URL,
// with data's keys and values urlencoded as the request body.
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}
// Head issues a HEAD to the specified URL. If the response is one of the
// following redirect codes, Head follows the redirect after calling the
// Client's CheckRedirect function.
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
//
// Head is a wrapper around DefaultClient.Head
func Head(url string) (resp *Response, err error) {
return DefaultClient.Head(url)
}
// Head issues a HEAD to the specified URL. If the response is one of the
// following redirect codes, Head follows the redirect after calling the
// Client's CheckRedirect function.
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
func (c *Client) Head(url string) (resp *Response, err error) {
req, err := NewRequest("HEAD", url, nil)
if err != nil {
return nil, err
}
return c.doFollowingRedirects(req, shouldRedirectGet)
}
// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
// 1) on Read EOF or Close, the timer t is Stopped,
// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and
// marked as net.Error that hit its timeout.
type cancelTimerBody struct {
t *time.Timer
rc io.ReadCloser
reqWasCanceled func() bool
}
func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
n, err = b.rc.Read(p)
if err == io.EOF {
b.t.Stop()
} else if err != nil && b.reqWasCanceled() {
return n, &httpError{
err: err.Error() + " (Client.Timeout exceeded while reading body)",
timeout: true,
}
}
return
}
func (b *cancelTimerBody) Close() error {
err := b.rc.Close()
b.t.Stop()
return err
}