benchmark: upgrade in-root benchmarks This CL updates all in-root benchmarks Updates #80564. Change-Id: I7749ec7106e6ee2284a13e1734e0096cd4a8fab0 Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/806160 LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com> Reviewed-by: Carlos Amedee <carlos@golang.org>
diff --git a/garbage/garbage.go b/garbage/garbage.go index fe185b6..d6fc829 100644 --- a/garbage/garbage.go +++ b/garbage/garbage.go
@@ -6,8 +6,8 @@ // It repeatedly parses net/http package with go/parser and then discards results. package main -// The source of net/http was captured at git tag go1.5.2 by -//go:generate sh -c "(echo 'package garbage'; echo 'var src = `'; bundle net/http http '' | sed 's/`/`+\"`\"+`/g'; echo '`') > nethttp.go" +// The source of net/http was captured from standard library net/http by +//go:generate sh -c "(echo 'package main'; echo ''; echo 'var src = `'; go run golang.org/x/tools/cmd/bundle@latest net/http | sed 's/`/`+\"`\"+`/g'; echo '`') > nethttp.go" import ( "fmt"
diff --git a/garbage/nethttp.go b/garbage/nethttp.go index 89c50e9..d3548f8 100644 --- a/garbage/nethttp.go +++ b/garbage/nethttp.go
@@ -1,12 +1,12 @@ package main var src = ` -// Code generated by golang.org/x/tools/cmd/bundle command: -// $ bundle net/http http +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +// $ bundle net/http // Package http provides HTTP client and server implementations. // -// Get, Head, Post, and PostForm make HTTP (or HTTPS) requests: +// [Get], [Head], [Post], and [PostForm] make HTTP (or HTTPS) requests: // // resp, err := http.Get("http://example.com/") // ... @@ -15,7 +15,7 @@ // resp, err := http.PostForm("http://example.com/form", // url.Values{"key": {"Value"}, "id": {"123"}}) // -// The client must close the response body when finished with it: +// The caller must close the response body when finished with it: // // resp, err := http.Get("http://example.com/") // if err != nil { @@ -25,8 +25,10 @@ // body, err := io.ReadAll(resp.Body) // // ... // +// # Clients and Transports +// // For control over HTTP client headers, redirect policy, and other -// settings, create a Client: +// settings, create a [Client]: // // client := &http.Client{ // CheckRedirect: redirectPolicyFunc, @@ -42,10 +44,11 @@ // // ... // // For control over proxies, TLS configuration, keep-alives, -// compression, and other settings, create a Transport: +// compression, and other settings, create a [Transport]: // // tr := &http.Transport{ -// TLSClientConfig: &tls.Config{RootCAs: pool}, +// MaxIdleConns: 10, +// IdleConnTimeout: 30 * time.Second, // DisableCompression: true, // } // client := &http.Client{Transport: tr} @@ -54,9 +57,11 @@ // Clients and Transports are safe for concurrent use by multiple // goroutines and for efficiency should only be created once and re-used. // +// # Servers +// // ListenAndServe starts an HTTP server with a given address and handler. -// The handler is usually nil, which means to use DefaultServeMux. -// Handle and HandleFunc add handlers to DefaultServeMux: +// The handler is usually nil, which means to use [DefaultServeMux]. +// [Handle] and [HandleFunc] add handlers to [DefaultServeMux]: // // http.Handle("/foo", fooHandler) // @@ -78,71 +83,140 @@ // } // log.Fatal(s.ListenAndServe()) // -package http +// # HTTP/2 +// +// The http package has transparent support for the HTTP/2 protocol. +// +// [Server] and [DefaultTransport] automatically enable HTTP/2 support +// when using HTTPS. [Transport] does not enable HTTP/2 by default. +// +// To enable or disable support for HTTP/1, HTTP/2, and/or unencrypted HTTP/2, +// see the [Server.Protocols] and [Transport.Protocols] configuration fields. +// +// To configure advanced HTTP/2 features, see the [Server.HTTP2] and +// [Transport.HTTP2] configuration fields. +// +// Alternatively, the following GODEBUG settings are currently supported: +// +// GODEBUG=http2client=0 # disable HTTP/2 client support +// GODEBUG=http2server=0 # disable HTTP/2 server support +// GODEBUG=http2debug=1 # enable verbose HTTP/2 debug logs +// GODEBUG=http2debug=2 # ... even more verbose, with frame dumps +// +// The "omithttp2" build tag may be used to disable the HTTP/2 implementation +// contained in the http package. +// + +package main import ( "bufio" "bytes" + "compress/flate" "compress/gzip" + "container/list" + "context" "crypto/tls" "encoding/base64" - "encoding/binary" "errors" "fmt" + "internal/godebug" "io" + "io/fs" "log" + "maps" + "math" + "math/rand/v2" "mime" "mime/multipart" "net" + "net/http/httptrace" "net/http/internal" + "net/http/internal/ascii" + "net/http/internal/http2" "net/textproto" "net/url" + urlpkg "net/url" "os" "path" "path/filepath" + "reflect" "runtime" + "slices" "sort" "strconv" "strings" "sync" "sync/atomic" "time" + "unicode" "unicode/utf8" + _ "unsafe" + + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http/httpproxy" + "golang.org/x/net/idna" ) -// A Client is an HTTP client. Its zero value (DefaultClient) is a -// usable client that uses DefaultTransport. +// A Client is an HTTP client. Its zero value ([DefaultClient]) is a +// usable client that uses [DefaultTransport]. // -// The Client's Transport typically has internal state (cached TCP +// The [Client.Transport] typically has internal state (cached TCP // connections), so Clients should be reused instead of created as // needed. Clients are safe for concurrent use by multiple goroutines. // -// A Client is higher-level than a RoundTripper (such as Transport) +// A Client is higher-level than a [RoundTripper] (such as [Transport]) // and additionally handles HTTP details such as cookies and // redirects. -type Client struct { +// +// When following redirects, the Client will forward all headers set on the +// initial [Request] except: +// +// - when forwarding sensitive headers like "Authorization", +// "WWW-Authenticate", and "Cookie" to untrusted targets. +// These headers will be ignored when following a redirect to a domain +// that is not a subdomain match or exact match of the initial domain. +// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" +// will forward the sensitive headers, but a redirect to "bar.com" will not. +// - when forwarding the "Cookie" header with a non-nil cookie Jar. +// Since each redirect may mutate the state of the cookie jar, +// a redirect may possibly alter a cookie set in the initial request. +// When forwarding the "Cookie" header, any mutated cookies will be omitted, +// with the expectation that the Jar will insert those mutated cookies +// with the updated values (assuming the origin matches). +// If Jar is nil, the initial cookies are forwarded without change. +type http_Client struct { // Transport specifies the mechanism by which individual // HTTP requests are made. // If nil, DefaultTransport is used. - Transport RoundTripper + Transport http_RoundTripper // CheckRedirect specifies the policy for handling redirects. // If CheckRedirect is not nil, the client calls it before // following an HTTP redirect. The arguments req and via are // the upcoming request and the requests made already, oldest // first. If CheckRedirect returns an error, the Client's Get - // method returns both the previous Response and - // CheckRedirect's error (wrapped in a url.Error) instead of - // issuing the Request req. + // method returns both the previous Response (with its Body + // closed) and CheckRedirect's error (wrapped in a url.Error) + // instead of issuing the Request req. + // As a special case, if CheckRedirect returns ErrUseLastResponse, + // then the most recent response is returned with its body + // unclosed, along with a nil error. // // If CheckRedirect is nil, the Client uses its default policy, // which is to stop after 10 consecutive requests. - CheckRedirect func(req *Request, via []*Request) error + CheckRedirect func(req *http_Request, via []*http_Request) error // Jar specifies the cookie jar. - // If Jar is nil, cookies are not sent in requests and ignored - // in responses. - Jar CookieJar + // + // The Jar is used to insert relevant cookies into every + // outbound Request and is updated with the cookie values + // of every inbound Response. The Jar is consulted for every + // redirect that the Client follows. + // + // If Jar is nil, cookies are only sent if they are explicitly + // set on the Request. + Jar http_CookieJar // Timeout specifies a time limit for requests made by this // Client. The timeout includes connection time, any @@ -152,362 +226,752 @@ // // A Timeout of zero means no timeout. // - // The Client's Transport must support the CancelRequest - // method or Client will return errors when attempting to make - // a request with Get, Head, Post, or Do. Client's default - // Transport (DefaultTransport) supports CancelRequest. + // The Client cancels requests to the underlying Transport + // as if the Request's Context ended. + // + // For compatibility, the Client will also use the deprecated + // CancelRequest method on Transport if found. New + // RoundTripper implementations should use the Request's Context + // for cancellation instead of implementing CancelRequest. Timeout time.Duration } -// DefaultClient is the default Client and is used by Get, Head, and Post. -var DefaultClient = &Client{} +// DefaultClient is the default [Client] and is used by [Get], [Head], and [Post]. +var http_DefaultClient = &http_Client{} // RoundTripper is an interface representing the ability to execute a -// single HTTP transaction, obtaining the Response for a given Request. +// single HTTP transaction, obtaining the [Response] for a given [Request]. // // A RoundTripper must be safe for concurrent use by multiple // goroutines. -type RoundTripper interface { +type http_RoundTripper interface { // RoundTrip executes a single HTTP transaction, returning - // the Response for the request req. RoundTrip should not - // attempt to interpret the response. In particular, - // RoundTrip must return err == nil if it obtained a response, - // regardless of the response's HTTP status code. A non-nil - // err should be reserved for failure to obtain a response. - // Similarly, RoundTrip should not attempt to handle - // higher-level protocol details such as redirects, + // a Response for the provided Request. + // + // RoundTrip should not attempt to interpret the response. In + // particular, RoundTrip must return err == nil if it obtained + // a response, regardless of the response's HTTP status code. + // A non-nil err should be reserved for failure to obtain a + // response. Similarly, RoundTrip should not attempt to + // handle higher-level protocol details such as redirects, // authentication, or cookies. // // RoundTrip should not modify the request, except for - // consuming and closing the Body, including on errors. The - // request's URL and Header fields are guaranteed to be - // initialized. - RoundTrip(*Request) (*Response, error) + // consuming and closing the Request's Body. RoundTrip may + // read fields of the request in a separate goroutine. Callers + // should not mutate or reuse the request until the Response's + // Body has been closed. + // + // RoundTrip must always close the body, including on errors, + // but depending on the implementation may do so in a separate + // goroutine even after RoundTrip returns. This means that + // callers wanting to reuse the body for subsequent requests + // must arrange to wait for the Close call before doing so. + // + // The Request's URL and Header fields must be initialized. + RoundTrip(*http_Request) (*http_Response, error) } -// Given a string of the form "host", "host:port", or "[ipv6::address]:port", -// return true if the string includes a port. -func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } - // refererForURL returns a referer without any authentication info or // an empty string if lastReq scheme is https and newReq scheme is http. -func refererForURL(lastReq, newReq *url.URL) string { - +// If the referer was explicitly set, then it will continue to be used. +func http_refererForURL(lastReq, newReq *url.URL, explicitRef string) string { + // https://tools.ietf.org/html/rfc7231#section-5.5.2 + // "Clients SHOULD NOT include a Referer header field in a + // (non-secure) HTTP request if the referring page was + // transferred with a secure protocol." if lastReq.Scheme == "https" && newReq.Scheme == "http" { return "" } + if explicitRef != "" { + return explicitRef + } + referer := lastReq.String() if lastReq.User != nil { - + // This is not very efficient, but is the best we can + // do without: + // - introducing a new method on URL + // - creating a race condition + // - copying the URL struct manually, which would cause + // maintenance problems down the line auth := lastReq.User.String() + "@" referer = strings.Replace(referer, auth, "", 1) } return referer } -// Used in Send to implement io.ReadCloser by bundling together the -// bufio.Reader through which we read the response, and the underlying -// network connection. -type readClose struct { - io.Reader - io.Closer -} - -func (c *Client) send(req *Request) (*Response, error) { +// didTimeout is non-nil only if err != nil. +func (c *http_Client) send(req *http_Request, deadline time.Time) (resp *http_Response, didTimeout func() bool, err error) { + cookieURL := req.URL + if req.Host != "" { + cookieURL = http_cloneURL(cookieURL) + cookieURL.Host = req.Host + } if c.Jar != nil { - for _, cookie := range c.Jar.Cookies(req.URL) { + for _, cookie := range c.Jar.Cookies(cookieURL) { req.AddCookie(cookie) } } - resp, err := send(req, c.transport()) + resp, didTimeout, err = http_send(req, c.transport(), deadline) if err != nil { - return nil, err + return nil, didTimeout, err } if c.Jar != nil { if rc := resp.Cookies(); len(rc) > 0 { - c.Jar.SetCookies(req.URL, rc) + c.Jar.SetCookies(cookieURL, rc) } } - return resp, err + return resp, nil, nil } -// Do sends an HTTP request and returns an HTTP response, following -// policy (e.g. redirects, cookies, auth) as configured on the client. -// -// An error is returned if caused by client policy (such as -// CheckRedirect), or if there was an HTTP protocol error. -// A non-2xx response doesn't cause an error. -// -// When err is nil, resp always contains a non-nil resp.Body. -// -// Callers should close resp.Body when done reading from it. If -// resp.Body is not closed, the Client's underlying RoundTripper -// (typically Transport) may not be able to re-use a persistent TCP -// connection to the server for a subsequent "keep-alive" request. -// -// The request Body, if non-nil, will be closed by the underlying -// Transport, even on errors. -// -// Generally Get, Post, or PostForm will be used instead of Do. -func (c *Client) Do(req *Request) (resp *Response, err error) { - if req.Method == "GET" || req.Method == "HEAD" { - return c.doFollowingRedirects(req, shouldRedirectGet) +func (c *http_Client) deadline() time.Time { + if c.Timeout > 0 { + return time.Now().Add(c.Timeout) } - if req.Method == "POST" || req.Method == "PUT" { - return c.doFollowingRedirects(req, shouldRedirectPost) - } - return c.send(req) + return time.Time{} } -func (c *Client) transport() RoundTripper { +func (c *http_Client) transport() http_RoundTripper { if c.Transport != nil { return c.Transport } - return DefaultTransport + return http_DefaultTransport } +// ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client. +var http_ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client") + // send issues an HTTP request. // Caller should close resp.Body when done reading from it. -func send(req *Request, t RoundTripper) (resp *Response, err error) { - if t == nil { +func http_send(ireq *http_Request, rt http_RoundTripper, deadline time.Time) (resp *http_Response, didTimeout func() bool, err error) { + req := ireq // req is either the original request, or a modified fork + + if rt == nil { req.closeBody() - return nil, errors.New("http: no Client.Transport or DefaultTransport") + return nil, http_alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") } if req.URL == nil { req.closeBody() - return nil, errors.New("http: nil Request.URL") + return nil, http_alwaysFalse, errors.New("http: nil Request.URL") } if req.RequestURI != "" { req.closeBody() - return nil, errors.New("http: Request.RequestURI can't be set in client requests.") + return nil, http_alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") } + // forkReq forks req into a shallow clone of ireq the first + // time it's called. + forkReq := func() { + if ireq == req { + req = new(http_Request) + *req = *ireq // shallow clone + } + } + + // Most the callers of send (Get, Post, et al) don't need + // Headers, leaving it uninitialized. We guarantee to the + // Transport that this has been initialized, though. if req.Header == nil { - req.Header = make(Header) + forkReq() + req.Header = make(http_Header) } if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { username := u.Username() password, _ := u.Password() - req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) + forkReq() + req.Header = http_cloneOrMakeHeader(ireq.Header) + req.Header.Set("Authorization", "Basic "+http_basicAuth(username, password)) } - resp, err = t.RoundTrip(req) + + if !deadline.IsZero() { + forkReq() + } + stopTimer, didTimeout := http_setRequestCancel(req, rt, deadline) + + resp, err = rt.RoundTrip(req) if err != nil { + stopTimer() if resp != nil { log.Printf("RoundTripper returned a response & error; ignoring response") } - return nil, err + if tlsErr, ok := err.(tls.RecordHeaderError); ok { + // If we get a bad TLS record header, check to see if the + // response looks like HTTP and give a more helpful error. + // See golang.org/issue/11111. + if string(tlsErr.RecordHeader[:]) == "HTTP/" { + err = http_ErrSchemeMismatch + } + } + return nil, didTimeout, err } - return resp, nil + if resp == nil { + return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) + } + if resp.Body == nil { + // The documentation on the Body field says “The http Client and Transport + // guarantee that Body is always non-nil, even on responses without a body + // or responses with a zero-length body.” Unfortunately, we didn't document + // that same constraint for arbitrary RoundTripper implementations, and + // RoundTripper implementations in the wild (mostly in tests) assume that + // they can use a nil Body to mean an empty one (similar to Request.Body). + // (See https://golang.org/issue/38095.) + // + // If the ContentLength allows the Body to be empty, fill in an empty one + // here to ensure that it is non-nil. + if resp.ContentLength > 0 && req.Method != "HEAD" { + return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) + } + resp.Body = io.NopCloser(strings.NewReader("")) + } + if !deadline.IsZero() { + resp.Body = &http_cancelTimerBody{ + stop: stopTimer, + rc: resp.Body, + reqDidTimeout: didTimeout, + } + } + return resp, nil, nil } -// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt +// timeBeforeContextDeadline reports whether the non-zero Time t is +// before ctx's deadline, if any. If ctx does not have a deadline, it +// always reports true (the deadline is considered infinite). +func http_timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { + d, ok := ctx.Deadline() + if !ok { + return true + } + return t.Before(d) +} + +// knownRoundTripperImpl reports whether rt is a RoundTripper that's +// maintained by the Go team and known to implement the latest +// optional semantics (notably contexts). The Request is used +// to check whether this particular request is using an alternate protocol, +// in which case we need to check the RoundTripper for that protocol. +func http_knownRoundTripperImpl(rt http_RoundTripper, req *http_Request) bool { + switch t := rt.(type) { + case *http_Transport: + if altRT := t.alternateRoundTripper(req); altRT != nil { + return http_knownRoundTripperImpl(altRT, req) + } + return true + case http_http2RoundTripper: + return true + } + // There's a very minor chance of a false positive with this. + // Instead of detecting our golang.org/x/net/http2.Transport, + // it might detect a Transport type in a different http2 + // package. But I know of none, and the only problem would be + // some temporarily leaked goroutines if the transport didn't + // support contexts. So this is a good enough heuristic: + if reflect.TypeOf(rt).String() == "*http2.Transport" { + return true + } + return false +} + +// setRequestCancel sets req.Cancel and adds a deadline context to req +// if deadline is non-zero. The RoundTripper's type is used to +// determine whether the legacy CancelRequest behavior should be used. +// +// As background, there are three ways to cancel a request: +// First was Transport.CancelRequest. (deprecated) +// Second was Request.Cancel. +// Third was Request.Context. +// This function populates the second and third, and uses the first if it really needs to. +func http_setRequestCancel(req *http_Request, rt http_RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { + if deadline.IsZero() { + return http_nop, http_alwaysFalse + } + knownTransport := http_knownRoundTripperImpl(rt, req) + oldCtx := req.Context() + + if req.Cancel == nil && knownTransport { + // If they already had a Request.Context that's + // expiring sooner, do nothing: + if !http_timeBeforeContextDeadline(deadline, oldCtx) { + return http_nop, http_alwaysFalse + } + + var cancelCtx func() + req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) + return cancelCtx, func() bool { return time.Now().After(deadline) } + } + initialReqCancel := req.Cancel // the user's original Request.Cancel, if any + + var cancelCtx func() + if http_timeBeforeContextDeadline(deadline, oldCtx) { + req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) + } + + cancel := make(chan struct{}) + req.Cancel = cancel + + doCancel := func() { + // The second way in the func comment above: + close(cancel) + // The first way, used only for RoundTripper + // implementations written before Go 1.5 or Go 1.6. + type canceler interface{ CancelRequest(*http_Request) } + if v, ok := rt.(canceler); ok { + v.CancelRequest(req) + } + } + + stopTimerCh := make(chan struct{}) + stopTimer = sync.OnceFunc(func() { + close(stopTimerCh) + if cancelCtx != nil { + cancelCtx() + } + }) + + timer := time.NewTimer(time.Until(deadline)) + var timedOut atomic.Bool + + go func() { + select { + case <-initialReqCancel: + doCancel() + timer.Stop() + case <-timer.C: + timedOut.Store(true) + doCancel() + case <-stopTimerCh: + timer.Stop() + } + }() + + return stopTimer, timedOut.Load +} + +// See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt // "To receive authorization, the client sends the userid and password, // separated by a single colon (":") character, within a base64 // encoded string in the credentials." // It is not meant to be urlencoded. -func basicAuth(username, password string) string { +func http_basicAuth(username, password string) string { auth := username + ":" + password return base64.StdEncoding.EncodeToString([]byte(auth)) } -// True if the specified HTTP status code is one for which the Get utility should -// automatically redirect. -func shouldRedirectGet(statusCode int) bool { - switch statusCode { - case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect: - return true - } - return false -} - -// True if the specified HTTP status code is one for which the Post utility should -// automatically redirect. -func shouldRedirectPost(statusCode int) bool { - switch statusCode { - case StatusFound, StatusSeeOther: - return true - } - return false -} - // Get issues a GET to the specified URL. If the response is one of // the following redirect codes, Get follows the redirect, up to a // maximum of 10 redirects: // -// 301 (Moved Permanently) -// 302 (Found) -// 303 (See Other) -// 307 (Temporary Redirect) +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) // // An error is returned if there were too many redirects or if there // was an HTTP protocol error. A non-2xx response doesn't cause an -// error. +// error. Any returned error will be of type [*url.Error]. The url.Error +// value's Timeout method will report true if the request timed out. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // // Get is a wrapper around DefaultClient.Get. // -// To make a request with custom headers, use NewRequest and +// To make a request with custom headers, use [NewRequest] and // DefaultClient.Do. -func Get(url string) (resp *Response, err error) { - return DefaultClient.Get(url) +// +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and DefaultClient.Do. +func http_Get(url string) (resp *http_Response, err error) { + return http_DefaultClient.Get(url) } // Get issues a GET to the specified URL. If the response is one of the // following redirect codes, Get follows the redirect after calling the -// Client's CheckRedirect function: +// [Client.CheckRedirect] function: // -// 301 (Moved Permanently) -// 302 (Found) -// 303 (See Other) -// 307 (Temporary Redirect) +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) // -// An error is returned if the Client's CheckRedirect function fails +// An error is returned if the [Client.CheckRedirect] function fails // or if there was an HTTP protocol error. A non-2xx response doesn't -// cause an error. +// cause an error. Any returned error will be of type [*url.Error]. The +// url.Error value's Timeout method will report true if the request +// timed out. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // -// To make a request with custom headers, use NewRequest and Client.Do. -func (c *Client) Get(url string) (resp *Response, err error) { - req, err := NewRequest("GET", url, nil) +// To make a request with custom headers, use [NewRequest] and [Client.Do]. +// +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and Client.Do. +func (c *http_Client) Get(url string) (resp *http_Response, err error) { + req, err := http_NewRequest("GET", url, nil) if err != nil { return nil, err } - return c.doFollowingRedirects(req, shouldRedirectGet) + return c.Do(req) } -func alwaysFalse() bool { return false } +func http_alwaysFalse() bool { return false } -func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) { - var base *url.URL - redirectChecker := c.CheckRedirect - if redirectChecker == nil { - redirectChecker = defaultCheckRedirect +// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to +// control how redirects are processed. If returned, the next request +// is not sent and the most recent response is returned with its body +// unclosed. +var http_ErrUseLastResponse = errors.New("net/http: use last response") + +// checkRedirect calls either the user's configured CheckRedirect +// function, or the default. +func (c *http_Client) checkRedirect(req *http_Request, via []*http_Request) error { + fn := c.CheckRedirect + if fn == nil { + fn = http_defaultCheckRedirect } - var via []*Request + return fn(req, via) +} - if ireq.URL == nil { - ireq.closeBody() - return nil, errors.New("http: nil Request.URL") - } +// redirectBehavior describes what should happen when the +// client encounters a 3xx status code from the server. +func http_redirectBehavior(reqMethod string, resp *http_Response, ireq *http_Request) (redirectMethod string, shouldRedirect, includeBody bool) { + switch resp.StatusCode { + case 301, 302, 303: + redirectMethod = reqMethod + shouldRedirect = true + includeBody = false - var reqmu sync.Mutex // guards req - req := ireq - - var timer *time.Timer - var atomicWasCanceled int32 // atomic bool (1 or 0) - var wasCanceled = alwaysFalse - if c.Timeout > 0 { - wasCanceled = func() bool { return atomic.LoadInt32(&atomicWasCanceled) != 0 } - type canceler interface { - CancelRequest(*Request) + // RFC 2616 allowed automatic redirection only with GET and + // HEAD requests. RFC 7231 lifts this restriction, but we still + // restrict other methods to GET to maintain compatibility. + // See Issue 18570. + if reqMethod != "GET" && reqMethod != "HEAD" { + redirectMethod = "GET" } - tr, ok := c.transport().(canceler) - if !ok { - return nil, fmt.Errorf("net/http: Client Transport of type %T doesn't support CancelRequest; Timeout not supported", c.transport()) - } - timer = time.AfterFunc(c.Timeout, func() { - atomic.StoreInt32(&atomicWasCanceled, 1) - reqmu.Lock() - defer reqmu.Unlock() - tr.CancelRequest(req) - }) - } + case 307, 308: + redirectMethod = reqMethod + shouldRedirect = true + includeBody = true - urlStr := "" - redirectFailed := false - for redirect := 0; ; redirect++ { - if redirect != 0 { - nreq := new(Request) - nreq.Method = ireq.Method - if ireq.Method == "POST" || ireq.Method == "PUT" { - nreq.Method = "GET" + if ireq.GetBody == nil && ireq.outgoingLength() != 0 { + // We had a request body, and 307/308 require + // re-sending it, but GetBody is not defined. So just + // return this response to the user instead of an + // error, like we did in Go 1.7 and earlier. + shouldRedirect = false + } + } + return redirectMethod, shouldRedirect, includeBody +} + +// urlErrorOp returns the (*url.Error).Op value to use for the +// provided (*Request).Method value. +func http_urlErrorOp(method string) string { + if method == "" { + return "Get" + } + if lowerMethod, ok := ascii.ToLower(method); ok { + return method[:1] + lowerMethod[1:] + } + return method +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (such as redirects, cookies, auth) as configured on the +// client. +// +// An error is returned if caused by client policy (such as +// CheckRedirect), or failure to speak HTTP (such as a network +// connectivity problem). A non-2xx status code doesn't cause an +// error. +// +// If the returned error is nil, the [Response] will contain a non-nil +// Body which the user is expected to close. If the Body is not both +// read to EOF and closed, the [Client]'s underlying [RoundTripper] +// (typically [Transport]) may not be able to re-use a persistent TCP +// connection to the server for a subsequent "keep-alive" request. +// Note, however, that [Transport] will automatically try to read a +// [Response] Body to EOF asynchronously up to a conservative limit +// when a Body is closed. +// +// The request Body, if non-nil, will be closed by the underlying +// Transport, even on errors. The Body may be closed asynchronously after +// Do returns. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when CheckRedirect fails, and even then +// the returned [Response.Body] is already closed. +// +// Generally [Get], [Post], or [PostForm] will be used instead of Do. +// +// If the server replies with a redirect, the Client first uses the +// CheckRedirect function to determine whether the redirect should be +// followed. If permitted, a 301, 302, or 303 redirect causes +// subsequent requests to use HTTP method GET +// (or HEAD if the original request was HEAD), with no body. +// A 307 or 308 redirect preserves the original HTTP method and body, +// provided that the [Request.GetBody] function is defined. +// The [NewRequest] function automatically sets GetBody for common +// standard library body types. +// +// Note that the [Client] redirect behavior does not follow the WHATWG +// Fetch standard. This is because it was written before established +// standards existed. As such, by modern standards, [Client] has a +// rather permissive behavior. For example, sensitive headers are +// retained on redirect to a subdomain or to a different scheme on the +// same host. +// +// Any returned error will be of type [*url.Error]. The url.Error +// value's Timeout method will report true if the request timed out. +func (c *http_Client) Do(req *http_Request) (*http_Response, error) { + return c.do(req) +} + +var http_testHookClientDoResult func(retres *http_Response, reterr error) + +func (c *http_Client) do(req *http_Request) (retres *http_Response, reterr error) { + if http_testHookClientDoResult != nil { + defer func() { http_testHookClientDoResult(retres, reterr) }() + } + if req.URL == nil { + req.closeBody() + return nil, &url.Error{ + Op: http_urlErrorOp(req.Method), + Err: errors.New("http: nil Request.URL"), + } + } + _ = *c // panic early if c is nil; see go.dev/issue/53521 + + var ( + deadline = c.deadline() + reqs []*http_Request + resp *http_Response + copyHeaders = c.makeHeadersCopier(req) + reqBodyClosed = false // have we closed the current req.Body? + + // Redirect behavior: + redirectMethod string + includeBody = true + stripSensitiveHeaders = false + ) + uerr := func(err error) error { + // the body may have been closed already by c.send() + if !reqBodyClosed { + req.closeBody() + } + var urlStr string + if resp != nil && resp.Request != nil { + urlStr = http_stripPassword(resp.Request.URL) + } else { + urlStr = http_stripPassword(req.URL) + } + return &url.Error{ + Op: http_urlErrorOp(reqs[0].Method), + URL: urlStr, + Err: err, + } + } + for { + // For all but the first request, create the next + // request hop and replace req. + if len(reqs) > 0 { + loc := resp.Header.Get("Location") + if loc == "" { + // While most 3xx responses include a Location, it is not + // required and 3xx responses without a Location have been + // observed in the wild. See issues #17773 and #49281. + return resp, nil } - nreq.Header = make(Header) - nreq.URL, err = base.Parse(urlStr) + u, err := req.URL.Parse(loc) if err != nil { - break + resp.closeBody() + return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) } - if len(via) > 0 { - - lastReq := via[len(via)-1] - if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" { - nreq.Header.Set("Referer", ref) + host := "" + if req.Host != "" && req.Host != req.URL.Host { + // If the caller specified a custom Host header and the + // redirect location is relative, preserve the Host header + // through the redirect. See issue #22233. + if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { + host = req.Host } - - err = redirectChecker(nreq, via) + } + ireq := reqs[0] + req = &http_Request{ + Method: redirectMethod, + Response: resp, + URL: u, + Header: make(http_Header), + Host: host, + Cancel: ireq.Cancel, + ctx: ireq.ctx, + } + if includeBody && ireq.GetBody != nil { + req.Body, err = ireq.GetBody() if err != nil { - redirectFailed = true - break + resp.closeBody() + return nil, uerr(err) + } + req.GetBody = ireq.GetBody + req.ContentLength = ireq.ContentLength + } + + // Copy original headers before setting the Referer, + // in case the user set Referer on their first request. + // If they really want to override, they can do it in + // their CheckRedirect func. + if !stripSensitiveHeaders && reqs[0].URL.Host != req.URL.Host { + if !http_shouldCopyHeaderOnRedirect(reqs[0].URL, req.URL) { + stripSensitiveHeaders = true } } - reqmu.Lock() - req = nreq - reqmu.Unlock() - } - - urlStr = req.URL.String() - if resp, err = c.send(req); err != nil { - if wasCanceled() { - err = &httpError{ - err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", - timeout: true, - } + copyHeaders(req, stripSensitiveHeaders, !includeBody) + // Add the Referer header from the most recent + // request URL to the new one, if it's not https->http: + if ref := http_refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" { + req.Header.Set("Referer", ref) } - break - } + err = c.checkRedirect(req, reqs) - if shouldRedirect(resp.StatusCode) { - // Read the body if small so underlying TCP connection will be re-used. - // No need to check for errors: if it fails, Transport won't reuse it anyway. + // Sentinel error to let users select the + // previous response, without closing its + // body. See Issue 10069. + if err == http_ErrUseLastResponse { + return resp, nil + } + + // Close the previous response's body. But + // read at least some of the body so if it's + // small the underlying TCP connection will be + // re-used. No need to check for errors: if it + // fails, the Transport won't reuse it anyway. const maxBodySlurpSize = 2 << 10 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) } resp.Body.Close() - if urlStr = resp.Header.Get("Location"); urlStr == "" { - err = fmt.Errorf("%d response missing Location header", resp.StatusCode) - break - } - base = req.URL - via = append(via, req) - continue - } - if timer != nil { - resp.Body = &cancelTimerBody{ - t: timer, - rc: resp.Body, - reqWasCanceled: wasCanceled, + + if err != nil { + // Special case for Go 1 compatibility: return both the response + // and an error if the CheckRedirect function failed. + // See https://golang.org/issue/3795 + // The resp.Body has already been closed. + ue := uerr(err) + ue.(*url.Error).URL = loc + return resp, ue } } - return resp, nil - } - method := ireq.Method - urlErr := &url.Error{ - Op: method[0:1] + strings.ToLower(method[1:]), - URL: urlStr, - Err: err, - } + reqs = append(reqs, req) + var err error + var didTimeout func() bool + if resp, didTimeout, err = c.send(req, deadline); err != nil { + // c.send() always closes req.Body + reqBodyClosed = true + if !deadline.IsZero() && didTimeout() { + err = &http_timeoutError{err.Error() + " (Client.Timeout exceeded while awaiting headers)"} + } + return nil, uerr(err) + } - if redirectFailed { + var shouldRedirect, includeBodyOnHop bool + redirectMethod, shouldRedirect, includeBodyOnHop = http_redirectBehavior(req.Method, resp, reqs[0]) + if !shouldRedirect { + return resp, nil + } + if !includeBodyOnHop { + // Once a hop drops the body, we never send it again + // (because we're now handling a redirect for a request with no body). + includeBody = false + } - return resp, urlErr + req.closeBody() } - - if resp != nil { - resp.Body.Close() - } - return nil, urlErr } -func defaultCheckRedirect(req *Request, via []*Request) error { +// makeHeadersCopier makes a function that copies headers from the +// initial Request, ireq. For every redirect, this function must be called +// so that it can copy headers into the upcoming Request. +func (c *http_Client) makeHeadersCopier(ireq *http_Request) func(req *http_Request, stripSensitiveHeaders, stripBodyHeaders bool) { + // The headers to copy are from the very initial request. + // We use a closured callback to keep a reference to these original headers. + var ( + ireqhdr = http_cloneOrMakeHeader(ireq.Header) + icookies map[string][]*http_Cookie + ) + if c.Jar != nil && ireq.Header.Get("Cookie") != "" { + icookies = make(map[string][]*http_Cookie) + for _, c := range ireq.Cookies() { + icookies[c.Name] = append(icookies[c.Name], c) + } + } + + return func(req *http_Request, stripSensitiveHeaders, stripBodyHeaders bool) { + // If Jar is present and there was some initial cookies provided + // via the request header, then we may need to alter the initial + // cookies as we follow redirects since each redirect may end up + // modifying a pre-existing cookie. + // + // Since cookies already set in the request header do not contain + // information about the original domain and path, the logic below + // assumes any new set cookies override the original cookie + // regardless of domain or path. + // + // See https://golang.org/issue/17494 + if c.Jar != nil && icookies != nil { + var changed bool + resp := req.Response // The response that caused the upcoming redirect + for _, c := range resp.Cookies() { + if _, ok := icookies[c.Name]; ok { + delete(icookies, c.Name) + changed = true + } + } + if changed { + ireqhdr.Del("Cookie") + var ss []string + for _, cs := range icookies { + for _, c := range cs { + ss = append(ss, c.Name+"="+c.Value) + } + } + slices.Sort(ss) // Ensure deterministic headers + ireqhdr.Set("Cookie", strings.Join(ss, "; ")) + } + } + + // Copy the initial request's Header values + // (at least the safe ones). + for k, vv := range ireqhdr { + sensitive := false + body := false + switch http_CanonicalHeaderKey(k) { + case "Authorization", "Www-Authenticate", "Cookie", "Cookie2", + "Proxy-Authorization", "Proxy-Authenticate": + sensitive = true + + case "Content-Encoding", "Content-Language", "Content-Location", + "Content-Type": + // Headers relating to the body which is removed for + // POST to GET redirects + // https://fetch.spec.whatwg.org/#http-redirect-fetch + body = true + + } + if !(sensitive && stripSensitiveHeaders) && !(body && stripBodyHeaders) { + req.Header[k] = vv + } + } + } +} + +func http_defaultCheckRedirect(req *http_Request, via []*http_Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") } @@ -518,125 +982,792 @@ // // Caller should close resp.Body when done reading from it. // -// If the provided body is an io.Closer, it is closed after the +// If the provided body is an [io.Closer], it is closed after the // request. // // Post is a wrapper around DefaultClient.Post. // -// To set custom headers, use NewRequest and DefaultClient.Do. -func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { - return DefaultClient.Post(url, bodyType, body) +// To set custom headers, use [NewRequest] and DefaultClient.Do. +// +// See the [Client.Do] method documentation for details on how redirects +// are handled. +// +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and DefaultClient.Do. +func http_Post(url, contentType string, body io.Reader) (resp *http_Response, err error) { + return http_DefaultClient.Post(url, contentType, body) } // Post issues a POST to the specified URL. // // Caller should close resp.Body when done reading from it. // -// If the provided body is an io.Closer, it is closed after the +// If the provided body is an [io.Closer], it is closed after the // request. // -// To set custom headers, use NewRequest and Client.Do. -func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { - req, err := NewRequest("POST", url, body) +// To set custom headers, use [NewRequest] and [Client.Do]. +// +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and [Client.Do]. +// +// See the [Client.Do] method documentation for details on how redirects +// are handled. +func (c *http_Client) Post(url, contentType string, body io.Reader) (resp *http_Response, err error) { + req, err := http_NewRequest("POST", url, body) if err != nil { return nil, err } - req.Header.Set("Content-Type", bodyType) - return c.doFollowingRedirects(req, shouldRedirectPost) + req.Header.Set("Content-Type", contentType) + return c.Do(req) } // PostForm issues a POST to the specified URL, with data's keys and // values URL-encoded as the request body. // // The Content-Type header is set to application/x-www-form-urlencoded. -// To set other headers, use NewRequest and DefaultClient.Do. +// To set other headers, use [NewRequest] and DefaultClient.Do. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // // PostForm is a wrapper around DefaultClient.PostForm. -func PostForm(url string, data url.Values) (resp *Response, err error) { - return DefaultClient.PostForm(url, data) +// +// See the [Client.Do] method documentation for details on how redirects +// are handled. +// +// To make a request with a specified [context.Context], use [NewRequestWithContext] +// and DefaultClient.Do. +func http_PostForm(url string, data url.Values) (resp *http_Response, err error) { + return http_DefaultClient.PostForm(url, data) } // PostForm issues a POST to the specified URL, // with data's keys and values URL-encoded as the request body. // // The Content-Type header is set to application/x-www-form-urlencoded. -// To set other headers, use NewRequest and DefaultClient.Do. +// To set other headers, use [NewRequest] and [Client.Do]. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. -func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { +// +// See the [Client.Do] method documentation for details on how redirects +// are handled. +// +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and Client.Do. +func (c *http_Client) PostForm(url string, data url.Values) (resp *http_Response, err error) { return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } -// Head issues a HEAD to the specified URL. If the response is one of +// Head issues a HEAD to the specified URL. If the response is one of // the following redirect codes, Head follows the redirect, up to a // maximum of 10 redirects: // -// 301 (Moved Permanently) -// 302 (Found) -// 303 (See Other) -// 307 (Temporary Redirect) +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) // -// Head is a wrapper around DefaultClient.Head -func Head(url string) (resp *Response, err error) { - return DefaultClient.Head(url) +// Head is a wrapper around DefaultClient.Head. +// +// To make a request with a specified [context.Context], use [NewRequestWithContext] +// and DefaultClient.Do. +func http_Head(url string) (resp *http_Response, err error) { + return http_DefaultClient.Head(url) } -// Head issues a HEAD to the specified URL. If the response is one of the +// Head issues a HEAD to the specified URL. If the response is one of the // following redirect codes, Head follows the redirect after calling the -// Client's CheckRedirect function: +// [Client.CheckRedirect] function: // -// 301 (Moved Permanently) -// 302 (Found) -// 303 (See Other) -// 307 (Temporary Redirect) -func (c *Client) Head(url string) (resp *Response, err error) { - req, err := NewRequest("HEAD", url, nil) +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) +// +// To make a request with a specified [context.Context], use [NewRequestWithContext] +// and [Client.Do]. +func (c *http_Client) Head(url string) (resp *http_Response, err error) { + req, err := http_NewRequest("HEAD", url, nil) if err != nil { return nil, err } - return c.doFollowingRedirects(req, shouldRedirectGet) + return c.Do(req) +} + +// CloseIdleConnections closes any connections on its [Transport] which +// were previously connected from previous requests but are now +// sitting idle in a "keep-alive" state. It does not interrupt any +// connections currently in use. +// +// If [Client.Transport] does not have a [Client.CloseIdleConnections] method +// then this method does nothing. +func (c *http_Client) CloseIdleConnections() { + type closeIdler interface { + CloseIdleConnections() + } + if tr, ok := c.transport().(closeIdler); ok { + tr.CloseIdleConnections() + } } // cancelTimerBody is an io.ReadCloser that wraps rc with two features: -// 1) on Read EOF or Close, the timer t is Stopped, -// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and -// marked as net.Error that hit its timeout. -type cancelTimerBody struct { - t *time.Timer - rc io.ReadCloser - reqWasCanceled func() bool +// 1. On Read error or close, the stop func is called. +// 2. On Read failure, if reqDidTimeout is true, the error is wrapped and +// marked as net.Error that hit its timeout. +type http_cancelTimerBody struct { + stop func() // stops the time.Timer waiting to cancel the request + rc io.ReadCloser + reqDidTimeout func() bool } -func (b *cancelTimerBody) Read(p []byte) (n int, err error) { +func (b *http_cancelTimerBody) Read(p []byte) (n int, err error) { n, err = b.rc.Read(p) - if err == io.EOF { - b.t.Stop() - } else if err != nil && b.reqWasCanceled() { - return n, &httpError{ - err: err.Error() + " (Client.Timeout exceeded while reading body)", - timeout: true, - } + if err == nil { + return n, nil } - return + if err == io.EOF { + return n, err + } + if b.reqDidTimeout() { + err = &http_timeoutError{err.Error() + " (Client.Timeout or context cancellation while reading body)"} + } + return n, err } -func (b *cancelTimerBody) Close() error { +func (b *http_cancelTimerBody) Close() error { err := b.rc.Close() - b.t.Stop() + b.stop() return err } +func http_shouldCopyHeaderOnRedirect(initial, dest *url.URL) bool { + // Permit sending auth/cookie headers from "foo.com" + // to "sub.foo.com". + + // Note that we don't send all cookies to subdomains + // automatically. This function is only used for + // Cookies set explicitly on the initial outgoing + // client request. Cookies automatically added via the + // CookieJar mechanism continue to follow each + // cookie's scope as set by Set-Cookie. But for + // outgoing requests with the Cookie header set + // directly, we don't know their scope, so we assume + // it's for *.domain.com. + + ihost, err1 := httpguts.PunycodeHostPort(initial.Hostname()) + dhost, err2 := httpguts.PunycodeHostPort(dest.Hostname()) + if err1 != nil || err2 != nil { + return false + } + ihost, ok1 := ascii.ToLower(ihost) + dhost, ok2 := ascii.ToLower(dhost) + return ok1 && ok2 && http_isDomainOrSubdomain(dhost, ihost) +} + +// isDomainOrSubdomain reports whether sub is a subdomain (or exact +// match) of the parent domain. +// +// Both domains must already be in canonical form. +func http_isDomainOrSubdomain(sub, parent string) bool { + if sub == parent { + return true + } + // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname). + // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone. + // For example, "::1%.www.example.com" is not a subdomain of "www.example.com". + if strings.ContainsAny(sub, ":%") { + return false + } + // If sub is "foo.example.com" and parent is "example.com", + // that means sub must end in "."+parent. + // Do it without allocating. + if !strings.HasSuffix(sub, parent) { + return false + } + return sub[len(sub)-len(parent)-1] == '.' +} + +func http_stripPassword(u *url.URL) string { + _, passSet := u.User.Password() + if passSet { + return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) + } + return u.String() +} + +// A ClientConn is a client connection to an HTTP server. +// +// Unlike a [Transport], a ClientConn represents a single connection. +// Most users should use a Transport rather than creating client connections directly. +type http_ClientConn struct { + cc http_genericClientConn + + stateHookMu sync.Mutex + userStateHook func(*http_ClientConn) + stateHookRunning bool + lastAvailable int + lastInFlight int + lastClosed bool +} + +// newClientConner is the interface implemented by HTTP/2 transports to create new client conns. +// +// The http package (this package) needs a way to ask the http2 package to +// create a client connection. +// +// Transport.TLSNextProto["h2"] contains a function which appears to do this, +// but for historical reasons it does not: The TLSNextProto function adds a +// *tls.Conn to the http2.Transport's connection pool and returns a RoundTripper +// which is backed by that connection pool. NewClientConn needs a way to get a +// single client connection out of the http2 package. +// +// The http2 package registers a RoundTripper with Transport.RegisterProtocol. +// If this RoundTripper implements newClientConner, then Transport.NewClientConn will use +// it to create new HTTP/2 client connections. +type http_newClientConner interface { + // NewClientConn creates a new client connection from a net.Conn. + // + // The RoundTripper returned by NewClientConn must implement genericClientConn. + // (We don't define NewClientConn as returning genericClientConn, + // because either we'd need to make genericClientConn an exported type + // or define it as a type alias. Neither is particularly appealing.) + // + // The state hook passed here is the internal state hook + // (ClientConn.maybeRunStateHook). The internal state hook calls + // the user state hook (if any), which is set by the user with + // ClientConn.SetStateHook. + // + // The client connection should arrange to call the internal state hook + // when the connection closes, when requests complete, and when the + // connection concurrency limit changes. + // + // The client connection must call the internal state hook when the connection state + // changes asynchronously, such as when a request completes. + // + // The internal state hook need not be called after synchronous changes to the state: + // Close, Reserve, Release, and RoundTrip calls which don't start a request + // do not need to call the hook. + // + // The general idea is that if we call (for example) Close, + // we know that the connection state has probably changed and we + // don't need the state hook to tell us that. + // However, if the connection closes asynchronously + // (because, for example, the other end of the conn closed it), + // the state hook needs to inform us. + NewClientConn(nc net.Conn, internalStateHook func()) (http_RoundTripper, error) +} + +// genericClientConn is an interface implemented by HTTP/2 client conns +// returned from newClientConner.NewClientConn. +// +// See the newClientConner doc comment for more information. +type http_genericClientConn interface { + Close() error + Err() error + RoundTrip(req *http_Request) (*http_Response, error) + Reserve() error + Release() + Available() int + InFlight() int +} + +// NewClientConn creates a new client connection to the given address. +// +// If scheme is "http", the connection is unencrypted. +// If scheme is "https", the connection uses TLS. +// +// The protocol used for the new connection is determined by the scheme, +// Transport.Protocols configuration field, and protocols supported by the +// server. See Transport.Protocols for more details. +// +// If Transport.Proxy is set and indicates that a request sent to the given +// address should use a proxy, the new connection uses that proxy. +// +// NewClientConn always creates a new connection, +// even if the Transport has an existing cached connection to the given host. +// +// The new connection is not added to the Transport's connection cache, +// and will not be used by [Transport.RoundTrip]. +// It does not count against the MaxIdleConns and MaxConnsPerHost limits. +// +// The caller is responsible for closing the new connection. +func (t *http_Transport) NewClientConn(ctx context.Context, scheme, address string) (*http_ClientConn, error) { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + + if t.h2Config != nil { + // Handle x/net/http2.Transport.NewClientConn passing us a net.Conn + // to create a ClientConn from. + if cc, err := t.http2NewClientConnFromContext(ctx); err != errors.ErrUnsupported { + return cc, err + } + } + + switch scheme { + case "http", "https": + default: + return nil, fmt.Errorf("net/http: invalid scheme %q", scheme) + } + + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + if port == "" { + port = http_schemePort(scheme) + } + + var proxyURL *url.URL + if t.Proxy != nil { + // Transport.Proxy takes a *Request, so create a fake one to pass it. + req := &http_Request{ + ctx: ctx, + Method: "GET", + URL: &url.URL{ + Scheme: scheme, + Host: host, + Path: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http_Header), + Body: http_NoBody, + Host: host, + } + var err error + proxyURL, err = t.Proxy(req) + if err != nil { + return nil, err + } + } + + cm := http_connectMethod{ + targetScheme: scheme, + targetAddr: net.JoinHostPort(host, port), + proxyURL: proxyURL, + } + + // The state hook is a bit tricky: + // The persistConn has a state hook which calls ClientConn.maybeRunStateHook, + // which in turn calls the user-provided state hook (if any). + // + // ClientConn.maybeRunStateHook handles debouncing hook calls for both + // HTTP/1 and HTTP/2. + // + // Since there's no need to change the persistConn's hook, we set it at creation time. + cc := &http_ClientConn{} + const isClientConn = true + pconn, err := t.dialConn(ctx, cm, isClientConn, cc.maybeRunStateHook) + if err != nil { + return nil, err + } + + // Note that cc.maybeRunStateHook may have been called + // in the short window between dialConn and now. + // This is fine. + cc.stateHookMu.Lock() + defer cc.stateHookMu.Unlock() + if pconn.alt != nil { + // If pconn.alt is set, this is a connection implemented in another package + // (probably x/net/http2) or the bundled copy in h2_bundle.go. + gc, ok := pconn.alt.(http_genericClientConn) + if !ok { + return nil, errors.New("http: NewClientConn returned something that is not a ClientConn") + } + cc.cc = gc + cc.lastAvailable = gc.Available() + } else { + // This is an HTTP/1 connection. + pconn.availch = make(chan struct{}, 1) + pconn.availch <- struct{}{} + cc.cc = http_http1ClientConn{pconn} + cc.lastAvailable = 1 + } + return cc, nil +} + +// Close closes the connection. +// Outstanding RoundTrip calls are interrupted. +func (cc *http_ClientConn) Close() error { + defer cc.maybeRunStateHook() + return cc.cc.Close() +} + +// Err reports any fatal connection errors. +// It returns nil if the connection is usable. +// If it returns non-nil, the connection can no longer be used. +func (cc *http_ClientConn) Err() error { + return cc.cc.Err() +} + +func http_validateClientConnRequest(req *http_Request) error { + if req.URL == nil { + return errors.New("http: nil Request.URL") + } + if req.Header == nil { + return errors.New("http: nil Request.Header") + } + // Validate the outgoing headers. + if err := http_validateHeaders(req.Header); err != "" { + return fmt.Errorf("http: invalid header %s", err) + } + // Validate the outgoing trailers too. + if err := http_validateHeaders(req.Trailer); err != "" { + return fmt.Errorf("http: invalid trailer %s", err) + } + if req.Method != "" && !http_validMethod(req.Method) { + return fmt.Errorf("http: invalid method %q", req.Method) + } + if req.URL.Host == "" { + return errors.New("http: no Host in request URL") + } + return nil +} + +// RoundTrip implements the [RoundTripper] interface. +// +// The request is sent on the client connection, +// regardless of the URL being requested or any proxy settings. +// +// If the connection is at its concurrency limit, +// RoundTrip waits for the connection to become available +// before sending the request. +func (cc *http_ClientConn) RoundTrip(req *http_Request) (*http_Response, error) { + defer cc.maybeRunStateHook() + if req.URL == nil && req.Method == ":ping" { + // Undocumented feature for sending a PING frame to a HTTP/2 connection, + // included to support x/net/http2.ClientConn.Ping. + pinger, ok := cc.cc.(interface { + Ping(context.Context) error + }) + if !ok { + return nil, errors.New("http: ClientConn does not support PING") + } + return nil, pinger.Ping(req.Context()) + } + if err := http_validateClientConnRequest(req); err != nil { + cc.Release() + return nil, err + } + return cc.cc.RoundTrip(req) +} + +// Available reports the number of requests that may be sent +// to the connection without blocking. +// It returns 0 if the connection is closed. +func (cc *http_ClientConn) Available() int { + return cc.cc.Available() +} + +// InFlight reports the number of requests in flight, +// including reserved requests. +// It returns 0 if the connection is closed. +func (cc *http_ClientConn) InFlight() int { + return cc.cc.InFlight() +} + +// Reserve reserves a concurrency slot on the connection. +// If Reserve returns nil, one additional RoundTrip call may be made +// without waiting for an existing request to complete. +// +// The reserved concurrency slot is accounted as an in-flight request. +// A successful call to RoundTrip will decrement the Available count +// and increment the InFlight count. +// +// Each successful call to Reserve should be followed by exactly one call +// to RoundTrip or Release, which will consume or release the reservation. +// +// If the connection is closed or at its concurrency limit, +// Reserve returns an error. +func (cc *http_ClientConn) Reserve() error { + defer cc.maybeRunStateHook() + return cc.cc.Reserve() +} + +// Release releases an unused concurrency slot reserved by Reserve. +// If there are no reserved concurrency slots, it has no effect. +func (cc *http_ClientConn) Release() { + defer cc.maybeRunStateHook() + cc.cc.Release() +} + +// shouldRunStateHook returns the user's state hook if we should call it, +// or nil if we don't need to call it at this time. +func (cc *http_ClientConn) shouldRunStateHook(stopRunning bool) func(*http_ClientConn) { + cc.stateHookMu.Lock() + defer cc.stateHookMu.Unlock() + if cc.cc == nil { + return nil + } + if stopRunning { + cc.stateHookRunning = false + } + if cc.userStateHook == nil { + return nil + } + if cc.stateHookRunning { + return nil + } + var ( + available = cc.Available() + inFlight = cc.InFlight() + closed = cc.Err() != nil + ) + var hook func(*http_ClientConn) + if available > cc.lastAvailable || inFlight < cc.lastInFlight || closed != cc.lastClosed { + hook = cc.userStateHook + cc.stateHookRunning = true + } + cc.lastAvailable = available + cc.lastInFlight = inFlight + cc.lastClosed = closed + return hook +} + +func (cc *http_ClientConn) maybeRunStateHook() { + hook := cc.shouldRunStateHook(false) + if hook == nil { + return + } + // Run the hook synchronously. + // + // This means that if, for example, the user calls resp.Body.Close to finish a request, + // the Close call will synchronously run the hook, giving the hook the chance to + // return the ClientConn to a connection pool before the next request is made. + hook(cc) + // The connection state may have changed while the hook was running, + // in which case we need to run it again. + // + // If we do need to run the hook again, do so in a new goroutine to avoid blocking + // the current goroutine indefinitely. + hook = cc.shouldRunStateHook(true) + if hook != nil { + go func() { + for hook != nil { + hook(cc) + hook = cc.shouldRunStateHook(true) + } + }() + } +} + +// SetStateHook arranges for f to be called when the state of the connection changes. +// At most one call to f is made at a time. +// If the connection's state has changed since it was created, +// f is called immediately in a separate goroutine. +// f may be called synchronously from RoundTrip or Response.Body.Close. +// +// If SetStateHook is called multiple times, the new hook replaces the old one. +// If f is nil, no further calls will be made to f after SetStateHook returns. +// +// f is called when Available increases (more requests may be sent on the connection), +// InFlight decreases (existing requests complete), or Err begins returning non-nil +// (the connection is no longer usable). +func (cc *http_ClientConn) SetStateHook(f func(*http_ClientConn)) { + cc.stateHookMu.Lock() + cc.userStateHook = f + cc.stateHookMu.Unlock() + cc.maybeRunStateHook() +} + +// http1ClientConn is a genericClientConn implementation backed by +// an HTTP/1 *persistConn (pconn.alt is nil). +type http_http1ClientConn struct { + pconn *http_persistConn +} + +func (cc http_http1ClientConn) RoundTrip(req *http_Request) (*http_Response, error) { + ctx := req.Context() + trace := httptrace.ContextClientTrace(ctx) + + // Convert Request.Cancel into context cancellation. + ctx, cancel := context.WithCancelCause(req.Context()) + if req.Cancel != nil { + go http_awaitLegacyCancel(ctx, cancel, req) + } + + treq := &http_transportRequest{http_Request: req, trace: trace, ctx: ctx, cancel: cancel} + resp, err := cc.pconn.roundTrip(treq) + if err != nil { + return nil, err + } + resp.Request = req + return resp, nil +} + +func (cc http_http1ClientConn) Close() error { + cc.pconn.close(errors.New("ClientConn closed")) + return nil +} + +func (cc http_http1ClientConn) Err() error { + select { + case <-cc.pconn.closech: + return cc.pconn.closed + default: + return nil + } +} + +func (cc http_http1ClientConn) Available() int { + cc.pconn.mu.Lock() + defer cc.pconn.mu.Unlock() + if cc.pconn.closed != nil || cc.pconn.reserved || cc.pconn.inFlight { + return 0 + } + return 1 +} + +func (cc http_http1ClientConn) InFlight() int { + cc.pconn.mu.Lock() + defer cc.pconn.mu.Unlock() + if cc.pconn.closed == nil && (cc.pconn.reserved || cc.pconn.inFlight) { + return 1 + } + return 0 +} + +func (cc http_http1ClientConn) Reserve() error { + cc.pconn.mu.Lock() + defer cc.pconn.mu.Unlock() + if cc.pconn.closed != nil { + return cc.pconn.closed + } + select { + case <-cc.pconn.availch: + default: + return errors.New("connection is unavailable") + } + cc.pconn.reserved = true + return nil +} + +func (cc http_http1ClientConn) Release() { + cc.pconn.mu.Lock() + defer cc.pconn.mu.Unlock() + if cc.pconn.reserved { + select { + case cc.pconn.availch <- struct{}{}: + default: + panic("cannot release reservation") + } + cc.pconn.reserved = false + } +} + +// cloneURLValues should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneURLValues +func http_cloneURLValues(v url.Values) url.Values { + if v == nil { + return nil + } + // http.Header and url.Values have the same representation, so temporarily + // treat it like http.Header, which does have a clone: + return url.Values(http_Header(v).Clone()) +} + +// cloneURL should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneURL +func http_cloneURL(u *url.URL) *url.URL { + return u.Clone() +} + +// cloneMultipartForm should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneMultipartForm +func http_cloneMultipartForm(f *multipart.Form) *multipart.Form { + if f == nil { + return nil + } + f2 := &multipart.Form{ + Value: (map[string][]string)(http_Header(f.Value).Clone()), + } + if f.File != nil { + m := make(map[string][]*multipart.FileHeader, len(f.File)) + for k, vv := range f.File { + vv2 := make([]*multipart.FileHeader, len(vv)) + for i, v := range vv { + vv2[i] = http_cloneMultipartFileHeader(v) + } + m[k] = vv2 + } + f2.File = m + } + return f2 +} + +// cloneMultipartFileHeader should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneMultipartFileHeader +func http_cloneMultipartFileHeader(fh *multipart.FileHeader) *multipart.FileHeader { + if fh == nil { + return nil + } + fh2 := new(multipart.FileHeader) + *fh2 = *fh + fh2.Header = textproto.MIMEHeader(http_Header(fh.Header).Clone()) + return fh2 +} + +// cloneOrMakeHeader invokes Header.Clone but if the +// result is nil, it'll instead make and return a non-nil Header. +// +// cloneOrMakeHeader should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneOrMakeHeader +func http_cloneOrMakeHeader(hdr http_Header) http_Header { + clone := hdr.Clone() + if clone == nil { + clone = make(http_Header) + } + return clone +} + +var http_httpcookiemaxnum = godebug.New("httpcookiemaxnum") + // A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an // HTTP response or the Cookie header of an HTTP request. // -// See http://tools.ietf.org/html/rfc6265 for details. -type Cookie struct { - Name string - Value string +// See https://tools.ietf.org/html/rfc6265 for details. +type http_Cookie struct { + Name string + Value string + Quoted bool // indicates whether the Value was originally quoted Path string // optional Domain string // optional @@ -646,199 +1777,386 @@ // MaxAge=0 means no 'Max-Age' attribute specified. // MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0' // MaxAge>0 means Max-Age attribute present and given in seconds - MaxAge int - Secure bool - HttpOnly bool - Raw string - Unparsed []string // Raw text of unparsed attribute-value pairs + MaxAge int + Secure bool + HttpOnly bool + SameSite http_SameSite + Partitioned bool + Raw string + Unparsed []string // Raw text of unparsed attribute-value pairs +} + +// SameSite allows a server to define a cookie attribute making it impossible for +// the browser to send this cookie along with cross-site requests. The main +// goal is to mitigate the risk of cross-origin information leakage, and provide +// some protection against cross-site request forgery attacks. +// +// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. +type http_SameSite int + +const ( + http_SameSiteDefaultMode http_SameSite = iota + 1 + http_SameSiteLaxMode + http_SameSiteStrictMode + http_SameSiteNoneMode +) + +var ( + http_errBlankCookie = errors.New("http: blank cookie") + http_errEqualNotFoundInCookie = errors.New("http: '=' not found in cookie") + http_errInvalidCookieName = errors.New("http: invalid cookie name") + http_errInvalidCookieValue = errors.New("http: invalid cookie value") + http_errCookieNumLimitExceeded = errors.New("http: number of cookies exceeded limit") +) + +const http_defaultCookieMaxNum = 3000 + +func http_cookieNumWithinMax(cookieNum int) bool { + withinDefaultMax := cookieNum <= http_defaultCookieMaxNum + if http_httpcookiemaxnum.Value() == "" { + return withinDefaultMax + } + if customMax, err := strconv.Atoi(http_httpcookiemaxnum.Value()); err == nil { + withinCustomMax := customMax == 0 || cookieNum <= customMax + if withinDefaultMax != withinCustomMax { + http_httpcookiemaxnum.IncNonDefault() + } + return withinCustomMax + } + return withinDefaultMax +} + +// ParseCookie parses a Cookie header value and returns all the cookies +// which were set in it. Since the same cookie name can appear multiple times +// the returned Values can contain more than one value for a given key. +func http_ParseCookie(line string) ([]*http_Cookie, error) { + nparts := strings.Count(line, ";") + 1 + if !http_cookieNumWithinMax(nparts) { + return nil, http_errCookieNumLimitExceeded + } else if nparts == 1 && textproto.TrimString(line) == "" { + return nil, http_errBlankCookie + } + cookies := make([]*http_Cookie, 0, nparts) + for s := range strings.SplitSeq(line, ";") { + s = textproto.TrimString(s) + name, value, found := strings.Cut(s, "=") + if !found { + return nil, http_errEqualNotFoundInCookie + } + if !http_isToken(name) { + return nil, http_errInvalidCookieName + } + value, quoted, found := http_parseCookieValue(value, true) + if !found { + return nil, http_errInvalidCookieValue + } + cookies = append(cookies, &http_Cookie{Name: name, Value: value, Quoted: quoted}) + } + return cookies, nil +} + +// ParseSetCookie parses a Set-Cookie header value and returns a cookie. +// It returns an error on syntax error. +func http_ParseSetCookie(line string) (*http_Cookie, error) { + parts := strings.Split(textproto.TrimString(line), ";") + if len(parts) == 1 && parts[0] == "" { + return nil, http_errBlankCookie + } + parts[0] = textproto.TrimString(parts[0]) + name, value, ok := strings.Cut(parts[0], "=") + if !ok { + return nil, http_errEqualNotFoundInCookie + } + name = textproto.TrimString(name) + if !http_isToken(name) { + return nil, http_errInvalidCookieName + } + value, quoted, ok := http_parseCookieValue(value, true) + if !ok { + return nil, http_errInvalidCookieValue + } + c := &http_Cookie{ + Name: name, + Value: value, + Quoted: quoted, + Raw: line, + } + for i := 1; i < len(parts); i++ { + parts[i] = textproto.TrimString(parts[i]) + if len(parts[i]) == 0 { + continue + } + + attr, val, _ := strings.Cut(parts[i], "=") + lowerAttr, isASCII := ascii.ToLower(attr) + if !isASCII { + continue + } + val, _, ok = http_parseCookieValue(val, false) + if !ok { + c.Unparsed = append(c.Unparsed, parts[i]) + continue + } + + switch lowerAttr { + case "samesite": + lowerVal, ascii := ascii.ToLower(val) + if !ascii { + c.SameSite = http_SameSiteDefaultMode + continue + } + switch lowerVal { + case "lax": + c.SameSite = http_SameSiteLaxMode + case "strict": + c.SameSite = http_SameSiteStrictMode + case "none": + c.SameSite = http_SameSiteNoneMode + default: + c.SameSite = http_SameSiteDefaultMode + } + continue + case "secure": + c.Secure = true + continue + case "httponly": + c.HttpOnly = true + continue + case "domain": + c.Domain = val + continue + case "max-age": + secs, err := strconv.Atoi(val) + if err != nil || secs != 0 && val[0] == '0' { + break + } + if secs <= 0 { + secs = -1 + } + c.MaxAge = secs + continue + case "expires": + c.RawExpires = val + exptime, err := time.Parse(time.RFC1123, val) + if err != nil { + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val) + if err != nil { + c.Expires = time.Time{} + break + } + } + c.Expires = exptime.UTC() + continue + case "path": + c.Path = val + continue + case "partitioned": + c.Partitioned = true + continue + } + c.Unparsed = append(c.Unparsed, parts[i]) + } + return c, nil } // readSetCookies parses all "Set-Cookie" values from // the header h and returns the successfully parsed Cookies. -func readSetCookies(h Header) []*Cookie { - cookies := []*Cookie{} +// +// If the amount of cookies exceeds CookieNumLimit, and httpcookielimitnum +// GODEBUG option is not explicitly turned off, this function will silently +// fail and return an empty slice. +func http_readSetCookies(h http_Header) []*http_Cookie { + cookieCount := len(h["Set-Cookie"]) + if cookieCount == 0 { + return []*http_Cookie{} + } + // Cookie limit was unfortunately introduced at a later point in time. + // As such, we can only fail by returning an empty slice rather than + // explicit error. + if !http_cookieNumWithinMax(cookieCount) { + return []*http_Cookie{} + } + cookies := make([]*http_Cookie, 0, cookieCount) for _, line := range h["Set-Cookie"] { - parts := strings.Split(strings.TrimSpace(line), ";") - if len(parts) == 1 && parts[0] == "" { - continue + if cookie, err := http_ParseSetCookie(line); err == nil { + cookies = append(cookies, cookie) } - parts[0] = strings.TrimSpace(parts[0]) - j := strings.Index(parts[0], "=") - if j < 0 { - continue - } - name, value := parts[0][:j], parts[0][j+1:] - if !isCookieNameValid(name) { - continue - } - value, success := parseCookieValue(value, true) - if !success { - continue - } - c := &Cookie{ - Name: name, - Value: value, - Raw: line, - } - for i := 1; i < len(parts); i++ { - parts[i] = strings.TrimSpace(parts[i]) - if len(parts[i]) == 0 { - continue - } - - attr, val := parts[i], "" - if j := strings.Index(attr, "="); j >= 0 { - attr, val = attr[:j], attr[j+1:] - } - lowerAttr := strings.ToLower(attr) - val, success = parseCookieValue(val, false) - if !success { - c.Unparsed = append(c.Unparsed, parts[i]) - continue - } - switch lowerAttr { - case "secure": - c.Secure = true - continue - case "httponly": - c.HttpOnly = true - continue - case "domain": - c.Domain = val - continue - case "max-age": - secs, err := strconv.Atoi(val) - if err != nil || secs != 0 && val[0] == '0' { - break - } - if secs <= 0 { - c.MaxAge = -1 - } else { - c.MaxAge = secs - } - continue - case "expires": - c.RawExpires = val - exptime, err := time.Parse(time.RFC1123, val) - if err != nil { - exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val) - if err != nil { - c.Expires = time.Time{} - break - } - } - c.Expires = exptime.UTC() - continue - case "path": - c.Path = val - continue - } - c.Unparsed = append(c.Unparsed, parts[i]) - } - cookies = append(cookies, c) } return cookies } -// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers. +// SetCookie adds a Set-Cookie header to the provided [ResponseWriter]'s headers. // The provided cookie must have a valid Name. Invalid cookies may be // silently dropped. -func SetCookie(w ResponseWriter, cookie *Cookie) { +func http_SetCookie(w http_ResponseWriter, cookie *http_Cookie) { if v := cookie.String(); v != "" { w.Header().Add("Set-Cookie", v) } } -// String returns the serialization of the cookie for use in a Cookie +// String returns the serialization of the cookie for use in a [Cookie] // header (if only Name and Value are set) or a Set-Cookie response // header (if other fields are set). // If c is nil or c.Name is invalid, the empty string is returned. -func (c *Cookie) String() string { - if c == nil || !isCookieNameValid(c.Name) { +func (c *http_Cookie) String() string { + if c == nil || !http_isToken(c.Name) { return "" } - var b bytes.Buffer - fmt.Fprintf(&b, "%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + // extraCookieLength derived from typical length of cookie attributes + // see RFC 6265 Sec 4.1. + const extraCookieLength = 110 + var b strings.Builder + b.Grow(len(c.Name) + len(c.Value) + len(c.Domain) + len(c.Path) + extraCookieLength) + b.WriteString(c.Name) + b.WriteRune('=') + b.WriteString(http_sanitizeCookieValue(c.Value, c.Quoted)) + if len(c.Path) > 0 { - fmt.Fprintf(&b, "; Path=%s", sanitizeCookiePath(c.Path)) + b.WriteString("; Path=") + b.WriteString(http_sanitizeCookiePath(c.Path)) } if len(c.Domain) > 0 { - if validCookieDomain(c.Domain) { - + if http_validCookieDomain(c.Domain) { + // A c.Domain containing illegal characters is not + // sanitized but simply dropped which turns the cookie + // into a host-only cookie. A leading dot is okay + // but won't be sent. d := c.Domain if d[0] == '.' { d = d[1:] } - fmt.Fprintf(&b, "; Domain=%s", d) + b.WriteString("; Domain=") + b.WriteString(d) } else { - log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", - c.Domain) + log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", c.Domain) } } - if c.Expires.Unix() > 0 { - fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(TimeFormat)) + var buf [len(http_TimeFormat)]byte + if http_validCookieExpires(c.Expires) { + b.WriteString("; Expires=") + b.Write(c.Expires.UTC().AppendFormat(buf[:0], http_TimeFormat)) } if c.MaxAge > 0 { - fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) + b.WriteString("; Max-Age=") + b.Write(strconv.AppendInt(buf[:0], int64(c.MaxAge), 10)) } else if c.MaxAge < 0 { - fmt.Fprintf(&b, "; Max-Age=0") + b.WriteString("; Max-Age=0") } if c.HttpOnly { - fmt.Fprintf(&b, "; HttpOnly") + b.WriteString("; HttpOnly") } if c.Secure { - fmt.Fprintf(&b, "; Secure") + b.WriteString("; Secure") + } + switch c.SameSite { + case http_SameSiteDefaultMode: + // Skip, default mode is obtained by not emitting the attribute. + case http_SameSiteNoneMode: + b.WriteString("; SameSite=None") + case http_SameSiteLaxMode: + b.WriteString("; SameSite=Lax") + case http_SameSiteStrictMode: + b.WriteString("; SameSite=Strict") + } + if c.Partitioned { + b.WriteString("; Partitioned") } return b.String() } +// Valid reports whether the cookie is valid. +func (c *http_Cookie) Valid() error { + if c == nil { + return errors.New("http: nil Cookie") + } + if !http_isToken(c.Name) { + return errors.New("http: invalid Cookie.Name") + } + if !c.Expires.IsZero() && !http_validCookieExpires(c.Expires) { + return errors.New("http: invalid Cookie.Expires") + } + for i := 0; i < len(c.Value); i++ { + if !http_validCookieValueByte(c.Value[i]) { + return fmt.Errorf("http: invalid byte %q in Cookie.Value", c.Value[i]) + } + } + if len(c.Path) > 0 { + for i := 0; i < len(c.Path); i++ { + if !http_validCookiePathByte(c.Path[i]) { + return fmt.Errorf("http: invalid byte %q in Cookie.Path", c.Path[i]) + } + } + } + if len(c.Domain) > 0 { + if !http_validCookieDomain(c.Domain) { + return errors.New("http: invalid Cookie.Domain") + } + } + if c.Partitioned { + if !c.Secure { + return errors.New("http: partitioned cookies must be set with Secure") + } + } + return nil +} + // readCookies parses all "Cookie" values from the header h and // returns the successfully parsed Cookies. // -// if filter isn't empty, only cookies of that name are returned -func readCookies(h Header, filter string) []*Cookie { - cookies := []*Cookie{} - lines, ok := h["Cookie"] - if !ok { - return cookies +// If filter isn't empty, only cookies of that name are returned. +// +// If the amount of cookies exceeds CookieNumLimit, and httpcookielimitnum +// GODEBUG option is not explicitly turned off, this function will silently +// fail and return an empty slice. +func http_readCookies(h http_Header, filter string) []*http_Cookie { + lines := h["Cookie"] + if len(lines) == 0 { + return []*http_Cookie{} } + // Cookie limit was unfortunately introduced at a later point in time. + // As such, we can only fail by returning an empty slice rather than + // explicit error. + cookieCount := 0 for _, line := range lines { - parts := strings.Split(strings.TrimSpace(line), ";") - if len(parts) == 1 && parts[0] == "" { - continue - } + cookieCount += strings.Count(line, ";") + 1 + } + if !http_cookieNumWithinMax(cookieCount) { + return []*http_Cookie{} + } - parsedPairs := 0 - for i := 0; i < len(parts); i++ { - parts[i] = strings.TrimSpace(parts[i]) - if len(parts[i]) == 0 { + cookies := make([]*http_Cookie, 0, len(lines)+strings.Count(lines[0], ";")) + for _, line := range lines { + line = textproto.TrimString(line) + + var part string + for len(line) > 0 { // continue since we have rest + part, line, _ = strings.Cut(line, ";") + part = textproto.TrimString(part) + if part == "" { continue } - name, val := parts[i], "" - if j := strings.Index(name, "="); j >= 0 { - name, val = name[:j], name[j+1:] - } - if !isCookieNameValid(name) { + name, val, _ := strings.Cut(part, "=") + name = textproto.TrimString(name) + if !http_isToken(name) { continue } if filter != "" && filter != name { continue } - val, success := parseCookieValue(val, true) - if !success { + val, quoted, ok := http_parseCookieValue(val, true) + if !ok { continue } - cookies = append(cookies, &Cookie{Name: name, Value: val}) - parsedPairs++ + cookies = append(cookies, &http_Cookie{Name: name, Value: val, Quoted: quoted}) } } return cookies } -// validCookieDomain returns whether v is a valid cookie domain-value. -func validCookieDomain(v string) bool { - if isCookieDomainName(v) { +// validCookieDomain reports whether v is a valid cookie domain-value. +func http_validCookieDomain(v string) bool { + if http_isCookieDomainName(v) { return true } if net.ParseIP(v) != nil && !strings.Contains(v, ":") { @@ -847,10 +2165,16 @@ return false } -// isCookieDomainName returns whether s is a valid domain name or a valid +// validCookieExpires reports whether v is a valid cookie expires-value. +func http_validCookieExpires(t time.Time) bool { + // IETF RFC 6265 Section 5.1.1.5, the year must not be less than 1601 + return t.Year() >= 1601 +} + +// isCookieDomainName reports whether s is a valid domain name or a valid // domain name with a leading dot '.'. It is almost a direct copy of // package net's isDomainName. -func isCookieDomainName(s string) bool { +func http_isCookieDomainName(s string) bool { if len(s) == 0 { return false } @@ -859,11 +2183,12 @@ } if s[0] == '.' { - + // A cookie domain attribute may start with a leading dot. + // Per RFC 6265 section 5.2.3, a leading dot is ignored. s = s[1:] } last := byte('.') - ok := false + ok := false // Ok once we've seen a letter. partlen := 0 for i := 0; i < len(s); i++ { c := s[i] @@ -871,20 +2196,20 @@ default: return false case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': - + // No '_' allowed here (in contrast to package net). ok = true partlen++ case '0' <= c && c <= '9': - + // fine partlen++ case c == '-': - + // Byte before dash cannot be dot. if last == '.' { return false } partlen++ case c == '.': - + // Byte before dot cannot be dot, dash. if last == '.' || last == '-' { return false } @@ -902,48 +2227,49 @@ return ok } -var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-") +var http_cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-") -func sanitizeCookieName(n string) string { - return cookieNameSanitizer.Replace(n) +func http_sanitizeCookieName(n string) string { + return http_cookieNameSanitizer.Replace(n) } -// http://tools.ietf.org/html/rfc6265#section-4.1.1 -// cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) -// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E -// ; US-ASCII characters excluding CTLs, -// ; whitespace DQUOTE, comma, semicolon, -// ; and backslash +// sanitizeCookieValue produces a suitable cookie-value from v. +// It receives a quoted bool indicating whether the value was originally +// quoted. +// https://tools.ietf.org/html/rfc6265#section-4.1.1 +// +// cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) +// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E +// ; US-ASCII characters excluding CTLs, +// ; whitespace DQUOTE, comma, semicolon, +// ; and backslash +// // We loosen this as spaces and commas are common in cookie values -// but we produce a quoted cookie-value in when value starts or ends -// with a comma or space. +// thus we produce a quoted cookie-value if v contains commas or spaces. // See https://golang.org/issue/7243 for the discussion. -func sanitizeCookieValue(v string) string { - v = sanitizeOrWarn("Cookie.Value", validCookieValueByte, v) - if len(v) == 0 { - return v - } - if v[0] == ' ' || v[0] == ',' || v[len(v)-1] == ' ' || v[len(v)-1] == ',' { +func http_sanitizeCookieValue(v string, quoted bool) string { + v = http_sanitizeOrWarn("Cookie.Value", http_validCookieValueByte, v) + if strings.ContainsAny(v, " ,") || quoted { return ` + "`" + `"` + "`" + ` + v + ` + "`" + `"` + "`" + ` } return v } -func validCookieValueByte(b byte) bool { +func http_validCookieValueByte(b byte) bool { return 0x20 <= b && b < 0x7f && b != '"' && b != ';' && b != '\\' } // path-av = "Path=" path-value // path-value = <any CHAR except CTLs or ";"> -func sanitizeCookiePath(v string) string { - return sanitizeOrWarn("Cookie.Path", validCookiePathByte, v) +func http_sanitizeCookiePath(v string) string { + return http_sanitizeOrWarn("Cookie.Path", http_validCookiePathByte, v) } -func validCookiePathByte(b byte) bool { +func http_validCookiePathByte(b byte) bool { return 0x20 <= b && b < 0x7f && b != ';' } -func sanitizeOrWarn(fieldName string, valid func(byte) bool, v string) string { +func http_sanitizeOrWarn(fieldName string, valid func(byte) bool, v string) string { ok := true for i := 0; i < len(v); i++ { if valid(v[i]) { @@ -965,51 +2291,282 @@ return string(buf) } -func parseCookieValue(raw string, allowDoubleQuote bool) (string, bool) { - +// parseCookieValue parses a cookie value according to RFC 6265. +// If allowDoubleQuote is true, parseCookieValue will consider that it +// is parsing the cookie-value; +// otherwise, it will consider that it is parsing a cookie-av value +// (cookie attribute-value). +// +// It returns the parsed cookie value, a boolean indicating whether the +// parsing was successful, and a boolean indicating whether the parsed +// value was enclosed in double quotes. +func http_parseCookieValue(raw string, allowDoubleQuote bool) (value string, quoted, ok bool) { + // Strip the quotes, if present. if allowDoubleQuote && len(raw) > 1 && raw[0] == '"' && raw[len(raw)-1] == '"' { raw = raw[1 : len(raw)-1] + quoted = true } for i := 0; i < len(raw); i++ { - if !validCookieValueByte(raw[i]) { - return "", false + if !http_validCookieValueByte(raw[i]) { + return "", quoted, false } } - return raw, true + return raw, quoted, true } -func isCookieNameValid(raw string) bool { - if raw == "" { - return false +// CrossOriginProtection implements protections against [Cross-Site Request +// Forgery (CSRF)] by rejecting non-safe cross-origin browser requests. +// +// Cross-origin requests are currently detected with the [Sec-Fetch-Site] +// header, available in all browsers since 2023, or by comparing the hostname of +// the [Origin] header with the Host header. +// +// The GET, HEAD, and OPTIONS methods are [safe methods] and are always allowed. +// It's important that applications do not perform any state changing actions +// due to requests with safe methods. +// +// Requests without Sec-Fetch-Site or Origin headers are currently assumed to be +// either same-origin or non-browser requests, and are allowed. +// +// The zero value of CrossOriginProtection is valid and has no trusted origins +// or bypass patterns. +// +// [Sec-Fetch-Site]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Site +// [Origin]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin +// [Cross-Site Request Forgery (CSRF)]: https://developer.mozilla.org/en-US/docs/Web/Security/Attacks/CSRF +// [safe methods]: https://developer.mozilla.org/en-US/docs/Glossary/Safe/HTTP +type http_CrossOriginProtection struct { + bypass atomic.Pointer[http_ServeMux] + trustedMu sync.RWMutex + trusted map[string]bool + deny atomic.Pointer[http_Handler] +} + +// NewCrossOriginProtection returns a new [CrossOriginProtection] value. +func http_NewCrossOriginProtection() *http_CrossOriginProtection { + return &http_CrossOriginProtection{} +} + +// AddTrustedOrigin allows all requests with an [Origin] header +// which exactly matches the given value. +// +// Origin header values are of the form "scheme://host[:port]". +// +// AddTrustedOrigin can be called concurrently with other methods +// or request handling, and applies to future requests. +// +// [Origin]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin +func (c *http_CrossOriginProtection) AddTrustedOrigin(origin string) error { + u, err := url.Parse(origin) + if err != nil { + return fmt.Errorf("invalid origin %q: %w", origin, err) } - return strings.IndexFunc(raw, isNotToken) < 0 + if u.Scheme == "" { + return fmt.Errorf("invalid origin %q: scheme is required", origin) + } + if u.Host == "" { + return fmt.Errorf("invalid origin %q: host is required", origin) + } + if u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("invalid origin %q: userinfo, path, query, and fragment are not allowed", origin) + } + c.trustedMu.Lock() + defer c.trustedMu.Unlock() + if c.trusted == nil { + c.trusted = make(map[string]bool) + } + c.trusted[origin] = true + return nil +} + +type http_noopHandler struct{} + +func (http_noopHandler) ServeHTTP(http_ResponseWriter, *http_Request) {} + +var http_sentinelHandler http_Handler = &http_noopHandler{} + +// AddInsecureBypassPattern permits all requests that match the given pattern. +// +// The pattern syntax and precedence rules are the same as [ServeMux]. Only +// requests that match the pattern directly are permitted. Those that ServeMux +// would redirect to a pattern (e.g. after cleaning the path or adding a +// trailing slash) are not. +// +// AddInsecureBypassPattern panics if the pattern conflicts with one already +// registered, or if the pattern is syntactically invalid (for example, an +// improperly formed wildcard). +// +// AddInsecureBypassPattern can be called concurrently with other methods or +// request handling, and applies to future requests. +func (c *http_CrossOriginProtection) AddInsecureBypassPattern(pattern string) { + var bypass *http_ServeMux + + // Lazily initialize c.bypass + for { + bypass = c.bypass.Load() + if bypass != nil { + break + } + bypass = http_NewServeMux() + if c.bypass.CompareAndSwap(nil, bypass) { + break + } + } + + bypass.Handle(pattern, http_sentinelHandler) +} + +// SetDenyHandler sets a handler to invoke when a request is rejected. +// The default error handler responds with a 403 Forbidden status. +// +// SetDenyHandler can be called concurrently with other methods +// or request handling, and applies to future requests. +// +// Check does not call the error handler. +func (c *http_CrossOriginProtection) SetDenyHandler(h http_Handler) { + if h == nil { + c.deny.Store(nil) + return + } + c.deny.Store(&h) +} + +// Check applies cross-origin checks to a request. +// It returns an error if the request should be rejected. +func (c *http_CrossOriginProtection) Check(req *http_Request) error { + switch req.Method { + case "GET", "HEAD", "OPTIONS": + // Safe methods are always allowed. + return nil + } + + switch req.Header.Get("Sec-Fetch-Site") { + case "": + // No Sec-Fetch-Site header is present. + // Fallthrough to check the Origin header. + case "same-origin", "none": + return nil + default: + if c.isRequestExempt(req) { + return nil + } + return http_errCrossOriginRequest + } + + origin := req.Header.Get("Origin") + if origin == "" { + // Neither Sec-Fetch-Site nor Origin headers are present. + // Either the request is same-origin or not a browser request. + return nil + } + + if o, err := url.Parse(origin); err == nil && o.Host == req.Host { + // The Origin header matches the Host header. Note that the Host header + // doesn't include the scheme, so we don't know if this might be an + // HTTP→HTTPS cross-origin request. We fail open, since all modern + // browsers support Sec-Fetch-Site since 2023, and running an older + // browser makes a clear security trade-off already. Sites can mitigate + // this with HTTP Strict Transport Security (HSTS). + return nil + } + + if c.isRequestExempt(req) { + return nil + } + return http_errCrossOriginRequestFromOldBrowser +} + +var ( + http_errCrossOriginRequest = errors.New("cross-origin request detected from Sec-Fetch-Site header") + http_errCrossOriginRequestFromOldBrowser = errors.New("cross-origin request detected, and/or browser is out of date: " + + "Sec-Fetch-Site is missing, and Origin does not match Host") +) + +// isRequestExempt checks the bypasses which require taking a lock, and should +// be deferred until the last moment. +func (c *http_CrossOriginProtection) isRequestExempt(req *http_Request) bool { + if bypass := c.bypass.Load(); bypass != nil { + if h, _ := bypass.Handler(req); h == http_sentinelHandler { + // The request matches a bypass pattern. + return true + } + } + + c.trustedMu.RLock() + defer c.trustedMu.RUnlock() + origin := req.Header.Get("Origin") + // The request matches a trusted origin. + return origin != "" && c.trusted[origin] +} + +// Handler returns a handler that applies cross-origin checks +// before invoking the handler h. +// +// If a request fails cross-origin checks, the request is rejected +// with a 403 Forbidden status or handled with the handler passed +// to [CrossOriginProtection.SetDenyHandler]. +func (c *http_CrossOriginProtection) Handler(h http_Handler) http_Handler { + return http_HandlerFunc(func(w http_ResponseWriter, r *http_Request) { + if err := c.Check(r); err != nil { + if deny := c.deny.Load(); deny != nil { + (*deny).ServeHTTP(w, r) + return + } + http_Error(w, err.Error(), http_StatusForbidden) + return + } + h.ServeHTTP(w, r) + }) } // fileTransport implements RoundTripper for the 'file' protocol. -type fileTransport struct { - fh fileHandler +type http_fileTransport struct { + fh http_fileHandler } -// NewFileTransport returns a new RoundTripper, serving the provided -// FileSystem. The returned RoundTripper ignores the URL host in its +// NewFileTransport returns a new [RoundTripper], serving the provided +// [FileSystem]. The returned RoundTripper ignores the URL host in its // incoming requests, as well as most other properties of the // request. // // The typical use case for NewFileTransport is to register the "file" -// protocol with a Transport, as in: +// protocol with a [Transport], as in: // -// t := &http.Transport{} -// t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) -// c := &http.Client{Transport: t} -// res, err := c.Get("file:///etc/passwd") -// ... -func NewFileTransport(fs FileSystem) RoundTripper { - return fileTransport{fileHandler{fs}} +// t := &http.Transport{} +// t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) +// c := &http.Client{Transport: t} +// res, err := c.Get("file:///etc/passwd") +// ... +func http_NewFileTransport(fs http_FileSystem) http_RoundTripper { + return http_fileTransport{http_fileHandler{fs}} } -func (t fileTransport) RoundTrip(req *Request) (resp *Response, err error) { +// NewFileTransportFS returns a new [RoundTripper], serving the provided +// file system fsys. The returned RoundTripper ignores the URL host in its +// incoming requests, as well as most other properties of the +// request. The files provided by fsys must implement [io.Seeker]. +// +// The typical use case for NewFileTransportFS is to register the "file" +// protocol with a [Transport], as in: +// +// fsys := os.DirFS("/") +// t := &http.Transport{} +// t.RegisterProtocol("file", http.NewFileTransportFS(fsys)) +// c := &http.Client{Transport: t} +// res, err := c.Get("file:///etc/passwd") +// ... +func http_NewFileTransportFS(fsys fs.FS) http_RoundTripper { + return http_NewFileTransport(http_FS(fsys)) +} - rw, resc := newPopulateResponseWriter() +func (t http_fileTransport) RoundTrip(req *http_Request) (resp *http_Response, err error) { + // We start ServeHTTP in a goroutine, which may take a long + // time if the file is large. The newPopulateResponseWriter + // call returns a channel which either ServeHTTP or finish() + // sends our *Response on, once the *Response itself has been + // populated (even if the body itself is still being + // written to the res.Body, a pipe) + rw, resc := http_newPopulateResponseWriter(req) go func() { t.fh.ServeHTTP(rw, req) rw.finish() @@ -1017,17 +2574,18 @@ return <-resc, nil } -func newPopulateResponseWriter() (*populateResponse, <-chan *Response) { +func http_newPopulateResponseWriter(req *http_Request) (*http_populateResponse, <-chan *http_Response) { pr, pw := io.Pipe() - rw := &populateResponse{ - ch: make(chan *Response), + rw := &http_populateResponse{ + ch: make(chan *http_Response), pw: pw, - res: &Response{ + res: &http_Response{ Proto: "HTTP/1.0", ProtoMajor: 1, - Header: make(Header), + Header: make(http_Header), Close: true, Body: pr, + Request: req, }, } return rw, rw.ch @@ -1037,16 +2595,16 @@ // in res, and writes its body to a pipe connected to the response // body. Once writes begin or finish() is called, the response is sent // on ch. -type populateResponse struct { - res *Response - ch chan *Response +type http_populateResponse struct { + res *http_Response + ch chan *http_Response wroteHeader bool hasContent bool sentResponse bool pw *io.PipeWriter } -func (pr *populateResponse) finish() { +func (pr *http_populateResponse) finish() { if !pr.wroteHeader { pr.WriteHeader(500) } @@ -1056,7 +2614,7 @@ pr.pw.Close() } -func (pr *populateResponse) sendResponse() { +func (pr *http_populateResponse) sendResponse() { if pr.sentResponse { return } @@ -1068,23 +2626,23 @@ pr.ch <- pr.res } -func (pr *populateResponse) Header() Header { +func (pr *http_populateResponse) Header() http_Header { return pr.res.Header } -func (pr *populateResponse) WriteHeader(code int) { +func (pr *http_populateResponse) WriteHeader(code int) { if pr.wroteHeader { return } pr.wroteHeader = true pr.res.StatusCode = code - pr.res.Status = fmt.Sprintf("%d %s", code, StatusText(code)) + pr.res.Status = fmt.Sprintf("%d %s", code, http_StatusText(code)) } -func (pr *populateResponse) Write(p []byte) (n int, err error) { +func (pr *http_populateResponse) Write(p []byte) (n int, err error) { if !pr.wroteHeader { - pr.WriteHeader(StatusOK) + pr.WriteHeader(http_StatusOK) } pr.hasContent = true if !pr.sentResponse { @@ -1093,28 +2651,72 @@ return pr.pw.Write(p) } -// A Dir implements FileSystem using the native file system restricted to a +// A Dir implements [FileSystem] using the native file system restricted to a // specific directory tree. // -// While the FileSystem.Open method takes '/'-separated paths, a Dir's string -// value is a filename on the native file system, not a URL, so it is separated -// by filepath.Separator, which isn't necessarily '/'. +// While the [FileSystem.Open] method takes '/'-separated paths, a Dir's string +// value is a directory path on the native file system, not a URL, so it is separated +// by [filepath.Separator], which isn't necessarily '/'. +// +// Note that Dir could expose sensitive files and directories. Dir will follow +// symlinks pointing out of the directory tree, which can be especially dangerous +// if serving from a directory in which users are able to create arbitrary symlinks. +// Dir will also allow access to files and directories starting with a period, +// which could expose sensitive directories like .git or sensitive files like +// .htpasswd. To exclude files with a leading period, remove the files/directories +// from the server or create a custom FileSystem implementation. // // An empty Dir is treated as ".". -type Dir string +type http_Dir string -func (d Dir) Open(name string) (File, error) { - if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || - strings.Contains(name, "\x00") { - return nil, errors.New("http: invalid character in file path") +// mapOpenError maps the provided non-nil error from opening name +// to a possibly better non-nil error. In particular, it turns OS-specific errors +// about opening files in non-directories into fs.ErrNotExist. See Issues 18984 and 49552. +func http_mapOpenError(originalErr error, name string, sep rune, stat func(string) (fs.FileInfo, error)) error { + if errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) { + return originalErr + } + + parts := strings.Split(name, string(sep)) + for i := range parts { + if parts[i] == "" { + continue + } + fi, err := stat(strings.Join(parts[:i+1], string(sep))) + if err != nil { + return originalErr + } + if !fi.IsDir() { + return fs.ErrNotExist + } + } + return originalErr +} + +// errInvalidUnsafePath is returned by Dir.Open when the call to +// filepath.Localize fails. filepath.Localize returns an error if the path +// cannot be represented by the operating system. +var http_errInvalidUnsafePath = errors.New("http: invalid or unsafe file path") + +// Open implements [FileSystem] using [os.Open], opening files for reading rooted +// and relative to the directory d. +func (d http_Dir) Open(name string) (http_File, error) { + path := path.Clean("/" + name)[1:] + if path == "" { + path = "." + } + path, err := filepath.Localize(path) + if err != nil { + return nil, http_errInvalidUnsafePath } dir := string(d) if dir == "" { dir = "." } - f, err := os.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + fullName := filepath.Join(dir, path) + f, err := os.Open(fullName) if err != nil { - return nil, err + return nil, http_mapOpenError(err, fullName, filepath.Separator, os.Stat) } return f, nil } @@ -1122,115 +2724,206 @@ // A FileSystem implements access to a collection of named files. // The elements in a file path are separated by slash ('/', U+002F) // characters, regardless of host operating system convention. -type FileSystem interface { - Open(name string) (File, error) +// See the [FileServer] function to convert a FileSystem to a [Handler]. +// +// This interface predates the [fs.FS] interface, which can be used instead: +// the [FS] adapter function converts an fs.FS to a FileSystem. +type http_FileSystem interface { + Open(name string) (http_File, error) } -// A File is returned by a FileSystem's Open method and can be -// served by the FileServer implementation. +// A File is returned by a [FileSystem]'s Open method and can be +// served by the [FileServer] implementation. // -// The methods should behave the same as those on an *os.File. -type File interface { +// The methods should behave the same as those on an [*os.File]. +type http_File interface { io.Closer io.Reader - Readdir(count int) ([]os.FileInfo, error) - Seek(offset int64, whence int) (int64, error) - Stat() (os.FileInfo, error) + io.Seeker + Readdir(count int) ([]fs.FileInfo, error) + Stat() (fs.FileInfo, error) } -func dirList(w ResponseWriter, f File) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - fmt.Fprintf(w, "<pre>\n") - for { - dirs, err := f.Readdir(100) - if err != nil || len(dirs) == 0 { - break - } - for _, d := range dirs { - name := d.Name() - if d.IsDir() { - name += "/" - } +type http_anyDirs interface { + len() int + name(i int) string + isDir(i int) bool +} - url := url.URL{Path: name} - fmt.Fprintf(w, "<a href=\"%s\">%s</a>\n", url.String(), htmlReplacer.Replace(name)) +type http_fileInfoDirs []fs.FileInfo + +func (d http_fileInfoDirs) len() int { return len(d) } + +func (d http_fileInfoDirs) isDir(i int) bool { return d[i].IsDir() } + +func (d http_fileInfoDirs) name(i int) string { return d[i].Name() } + +type http_dirEntryDirs []fs.DirEntry + +func (d http_dirEntryDirs) len() int { return len(d) } + +func (d http_dirEntryDirs) isDir(i int) bool { return d[i].IsDir() } + +func (d http_dirEntryDirs) name(i int) string { return d[i].Name() } + +func http_dirList(w http_ResponseWriter, r *http_Request, f http_File) { + // Prefer to use ReadDir instead of Readdir, + // because the former doesn't require calling + // Stat on every entry of a directory on Unix. + var dirs http_anyDirs + var err error + if d, ok := f.(fs.ReadDirFile); ok { + var list http_dirEntryDirs + list, err = d.ReadDir(-1) + dirs = list + } else { + var list http_fileInfoDirs + list, err = f.Readdir(-1) + dirs = list + } + + if err != nil { + http_logf(r, "http: error reading directory: %v", err) + http_Error(w, "Error reading directory", http_StatusInternalServerError) + return + } + sort.Slice(dirs, func(i, j int) bool { return dirs.name(i) < dirs.name(j) }) + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprintf(w, "<!doctype html>\n") + fmt.Fprintf(w, "<meta name=\"viewport\" content=\"width=device-width\">\n") + fmt.Fprintf(w, "<pre>\n") + for i, n := 0, dirs.len(); i < n; i++ { + name := dirs.name(i) + if dirs.isDir(i) { + name += "/" } + // name may contain '?' or '#', which must be escaped to remain + // part of the URL path, and not indicate the start of a query + // string or fragment. + url := url.URL{Path: name} + fmt.Fprintf(w, "<a href=\"%s\">%s</a>\n", url.String(), http_htmlReplacer.Replace(name)) } fmt.Fprintf(w, "</pre>\n") } +// GODEBUG=httpservecontentkeepheaders=1 restores the pre-1.23 behavior of not deleting +// Cache-Control, Content-Encoding, Etag, or Last-Modified headers on ServeContent errors. +var http_httpservecontentkeepheaders = godebug.New("httpservecontentkeepheaders") + +// serveError serves an error from ServeFile, ServeFileFS, and ServeContent. +// Because those can all be configured by the caller by setting headers like +// Etag, Last-Modified, and Cache-Control to send on a successful response, +// the error path needs to clear them, since they may not be meant for errors. +func http_serveError(w http_ResponseWriter, text string, code int) { + h := w.Header() + + nonDefault := false + for _, k := range []string{ + "Cache-Control", + "Content-Encoding", + "Etag", + "Last-Modified", + } { + if !h.has(k) { + continue + } + if http_httpservecontentkeepheaders.Value() == "1" { + nonDefault = true + } else { + h.Del(k) + } + } + if nonDefault { + http_httpservecontentkeepheaders.IncNonDefault() + } + + http_Error(w, text, code) +} + // ServeContent replies to the request using the content in the -// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// provided ReadSeeker. The main benefit of ServeContent over [io.Copy] // is that it handles Range requests properly, sets the MIME type, and -// handles If-Modified-Since requests. +// handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, +// and If-Range requests. // // If the response's Content-Type header is not set, ServeContent // first tries to deduce the type from name's file extension and, // if that fails, falls back to reading the first block of the content -// and passing it to DetectContentType. +// and passing it to [DetectContentType]. // The name is otherwise unused; in particular it can be empty and is // never sent in the response. // // If modtime is not the zero time or Unix epoch, ServeContent -// includes it in a Last-Modified header in the response. If the +// includes it in a Last-Modified header in the response. If the // request includes an If-Modified-Since header, ServeContent uses // modtime to decide whether the content needs to be sent at all. // // The content's Seek method must work: ServeContent uses // a seek to the end of the content to determine its size. +// Note that [*os.File] implements the [io.ReadSeeker] interface. // -// If the caller has set w's ETag header, ServeContent uses it to -// handle requests using If-Range and If-None-Match. +// If the caller has set w's ETag header formatted per RFC 7232, section 2.3, +// ServeContent uses it to handle requests using If-Match, If-None-Match, or If-Range. // -// Note that *os.File implements the io.ReadSeeker interface. -func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { +// If an error occurs when serving the request (for example, when +// handling an invalid range request), ServeContent responds with an +// error message. By default, ServeContent strips the Cache-Control, +// Content-Encoding, ETag, and Last-Modified headers from error responses. +// The GODEBUG setting httpservecontentkeepheaders=1 causes ServeContent +// to preserve these headers. +func http_ServeContent(w http_ResponseWriter, req *http_Request, name string, modtime time.Time, content io.ReadSeeker) { sizeFunc := func() (int64, error) { - size, err := content.Seek(0, os.SEEK_END) + size, err := content.Seek(0, io.SeekEnd) if err != nil { - return 0, errSeeker + return 0, http_errSeeker } - _, err = content.Seek(0, os.SEEK_SET) + _, err = content.Seek(0, io.SeekStart) if err != nil { - return 0, errSeeker + return 0, http_errSeeker } return size, nil } - serveContent(w, req, name, modtime, sizeFunc, content) + http_serveContent(w, req, name, modtime, sizeFunc, content) } // errSeeker is returned by ServeContent's sizeFunc when the content // doesn't seek properly. The underlying Seeker's error text isn't // included in the sizeFunc reply so it's not sent over HTTP to end // users. -var errSeeker = errors.New("seeker can't seek") +var http_errSeeker = errors.New("seeker can't seek") + +// errNoOverlap is returned by serveContent's parseRange if first-byte-pos of +// all of the byte-range-spec values is greater than the content size. +var http_errNoOverlap = errors.New("invalid range: failed to overlap") // if name is empty, filename is unknown. (used for mime type, before sniffing) // if modtime.IsZero(), modtime is unknown. // content must be seeked to the beginning of the file. // The sizeFunc is called at most once. Its error, if any, is sent in the HTTP response. -func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) { - if checkLastModified(w, r, modtime) { - return - } - rangeReq, done := checkETag(w, r, modtime) +func http_serveContent(w http_ResponseWriter, r *http_Request, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) { + http_setLastModified(w, modtime) + done, rangeReq := http_checkPreconditions(w, r, modtime) if done { return } - code := StatusOK + code := http_StatusOK + // If Content-Type isn't set, use the file's extension to find it, but + // if the Content-Type is unset explicitly, do not sniff the type. ctypes, haveType := w.Header()["Content-Type"] var ctype string if !haveType { ctype = mime.TypeByExtension(filepath.Ext(name)) if ctype == "" { // read a chunk to decide between utf-8 text and binary - var buf [sniffLen]byte + var buf [internal.SniffLen]byte n, _ := io.ReadFull(content, buf[:]) - ctype = DetectContentType(buf[:n]) - _, err := content.Seek(0, os.SEEK_SET) + ctype = http_DetectContentType(buf[:n]) + _, err := content.Seek(0, io.SeekStart) // rewind to output whole file if err != nil { - Error(w, "seeker can't seek", StatusInternalServerError) + http_serveError(w, "seeker can't seek", http_StatusInternalServerError) return } } @@ -1241,69 +2934,125 @@ size, err := sizeFunc() if err != nil { - Error(w, err.Error(), StatusInternalServerError) + http_serveError(w, err.Error(), http_StatusInternalServerError) + return + } + if size < 0 { + // Should never happen but just to be sure + http_serveError(w, "negative content size computed", http_StatusInternalServerError) return } + // handle Content-Range header. sendSize := size var sendContent io.Reader = content - if size >= 0 { - ranges, err := parseRange(rangeReq, size) - if err != nil { - Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) - return - } - if sumRangesSize(ranges) > size { - + ranges, err := http_parseRange(rangeReq, size) + switch err { + case nil: + case http_errNoOverlap: + if size == 0 { + // Some clients add a Range header to all requests to + // limit the size of the response. If the file is empty, + // ignore the range header and respond with a 200 rather + // than a 416. ranges = nil + break } - switch { - case len(ranges) == 1: - - ra := ranges[0] - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { - Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) - return - } - sendSize = ra.length - code = StatusPartialContent - w.Header().Set("Content-Range", ra.contentRange(size)) - case len(ranges) > 1: - sendSize = rangesMIMESize(ranges, ctype, size) - code = StatusPartialContent - - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) - sendContent = pr - defer pr.Close() - go func() { - for _, ra := range ranges { - part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { - pw.CloseWithError(err) - return - } - if _, err := io.CopyN(part, content, ra.length); err != nil { - pw.CloseWithError(err) - return - } - } - mw.Close() - pw.Close() - }() - } - - w.Header().Set("Accept-Ranges", "bytes") - if w.Header().Get("Content-Encoding") == "" { - w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) - } + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) + fallthrough + default: + http_serveError(w, err.Error(), http_StatusRequestedRangeNotSatisfiable) + return } + if http_sumRangesSize(ranges) > size { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + ranges = nil + } + switch { + case len(ranges) == 1: + // RFC 7233, Section 4.1: + // "If a single part is being transferred, the server + // generating the 206 response MUST generate a + // Content-Range header field, describing what range + // of the selected representation is enclosed, and a + // payload consisting of the range. + // ... + // A server MUST NOT generate a multipart response to + // a request for a single range, since a client that + // does not request multiple parts might not support + // multipart responses." + ra := ranges[0] + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { + http_serveError(w, err.Error(), http_StatusRequestedRangeNotSatisfiable) + return + } + sendSize = ra.length + code = http_StatusPartialContent + w.Header().Set("Content-Range", ra.contentRange(size)) + case len(ranges) > 1: + sendSize = http_rangesMIMESize(ranges, ctype, size) + code = http_StatusPartialContent + + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) + sendContent = pr + defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. + go func() { + for _, ra := range ranges { + part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.CopyN(part, content, ra.length); err != nil { + pw.CloseWithError(err) + return + } + } + mw.Close() + pw.Close() + }() + } + + w.Header().Set("Accept-Ranges", "bytes") + + // We should be able to unconditionally set the Content-Length here. + // + // However, there is a pattern observed in the wild that this breaks: + // The user wraps the ResponseWriter in one which gzips data written to it, + // and sets "Content-Encoding: gzip". + // + // The user shouldn't be doing this; the serveContent path here depends + // on serving seekable data with a known length. If you want to compress + // on the fly, then you shouldn't be using ServeFile/ServeContent, or + // you should compress the entire file up-front and provide a seekable + // view of the compressed data. + // + // However, since we've observed this pattern in the wild, and since + // setting Content-Length here breaks code that mostly-works today, + // skip setting Content-Length if the user set Content-Encoding. + // + // If this is a range request, always set Content-Length. + // If the user isn't changing the bytes sent in the ResponseWrite, + // the Content-Length will be correct. + // If the user is changing the bytes sent, then the range request wasn't + // going to work properly anyway and we aren't worse off. + // + // A possible future improvement on this might be to look at the type + // of the ResponseWriter, and always set Content-Length if it's one + // that we recognize. + if len(ranges) > 0 || w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } w.WriteHeader(code) if r.Method != "HEAD" { @@ -1311,137 +3060,335 @@ } } -var unixEpochTime = time.Unix(0, 0) - -// modtime is the modification time of the resource to be served, or IsZero(). -// return value is whether this request is now complete. -func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool { - if modtime.IsZero() || modtime.Equal(unixEpochTime) { - - return false +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func http_scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 } - - if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - w.WriteHeader(StatusNotModified) - return true + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" } - w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) - return false + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" } -// checkETag implements If-None-Match and If-Range checks. -// -// The ETag or modtime must have been previously set in the -// ResponseWriter's headers. The modtime is only compared at second -// granularity and may be the zero value to mean unknown. -// -// The return value is the effective request "Range" header to use and -// whether this request is now considered done. -func checkETag(w ResponseWriter, r *Request, modtime time.Time) (rangeReq string, done bool) { - etag := w.Header().get("Etag") - rangeReq = r.Header.get("Range") +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func http_etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} - if ir := r.Header.get("If-Range"); ir != "" && ir != etag { +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func http_etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} - timeMatches := false - if !modtime.IsZero() { - if t, err := ParseTime(ir); err == nil && t.Unix() == modtime.Unix() { - timeMatches = true - } - } - if !timeMatches { - rangeReq = "" - } +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type http_condResult int + +const ( + http_condNone http_condResult = iota + http_condTrue + http_condFalse +) + +func http_checkIfMatch(w http_ResponseWriter, r *http_Request) http_condResult { + im := r.Header.Get("If-Match") + if im == "" { + return http_condNone } - - if inm := r.Header.get("If-None-Match"); inm != "" { - + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return http_condTrue + } + etag, remain := http_scanETag(im) if etag == "" { - return rangeReq, false + break } - - if r.Method != "GET" && r.Method != "HEAD" { - return rangeReq, false + if http_etagStrongMatch(etag, w.Header().get("Etag")) { + return http_condTrue } + im = remain + } - if inm == etag || inm == "*" { - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - w.WriteHeader(StatusNotModified) - return "", true + return http_condFalse +} + +func http_checkIfUnmodifiedSince(r *http_Request, modtime time.Time) http_condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" || http_isZeroTime(modtime) { + return http_condNone + } + t, err := http_ParseTime(ius) + if err != nil { + return http_condNone + } + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return http_condTrue + } + return http_condFalse +} + +func http_checkIfNoneMatch(w http_ResponseWriter, r *http_Request) http_condResult { + inm := r.Header.get("If-None-Match") + if inm == "" { + return http_condNone + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + continue + } + if buf[0] == '*' { + return http_condFalse + } + etag, remain := http_scanETag(buf) + if etag == "" { + break + } + if http_etagWeakMatch(etag, w.Header().get("Etag")) { + return http_condFalse + } + buf = remain + } + return http_condTrue +} + +func http_checkIfModifiedSince(r *http_Request, modtime time.Time) http_condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return http_condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || http_isZeroTime(modtime) { + return http_condNone + } + t, err := http_ParseTime(ims) + if err != nil { + return http_condNone + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return http_condFalse + } + return http_condTrue +} + +func http_checkIfRange(w http_ResponseWriter, r *http_Request, modtime time.Time) http_condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return http_condNone + } + ir := r.Header.get("If-Range") + if ir == "" { + return http_condNone + } + etag, _ := http_scanETag(ir) + if etag != "" { + if http_etagStrongMatch(etag, w.Header().Get("Etag")) { + return http_condTrue + } else { + return http_condFalse } } - return rangeReq, false + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return http_condFalse + } + t, err := http_ParseTime(ir) + if err != nil { + return http_condFalse + } + if t.Unix() == modtime.Unix() { + return http_condTrue + } + return http_condFalse +} + +var http_unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func http_isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(http_unixEpochTime) +} + +func http_setLastModified(w http_ResponseWriter, modtime time.Time) { + if !http_isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http_TimeFormat)) + } +} + +func http_writeNotModified(w http_ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + delete(h, "Content-Encoding") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(http_StatusNotModified) +} + +// checkPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. +func http_checkPreconditions(w http_ResponseWriter, r *http_Request, modtime time.Time) (done bool, rangeHeader string) { + // This function carefully follows RFC 7232 section 6. + ch := http_checkIfMatch(w, r) + if ch == http_condNone { + ch = http_checkIfUnmodifiedSince(r, modtime) + } + if ch == http_condFalse { + w.WriteHeader(http_StatusPreconditionFailed) + return true, "" + } + switch http_checkIfNoneMatch(w, r) { + case http_condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + http_writeNotModified(w) + return true, "" + } else { + w.WriteHeader(http_StatusPreconditionFailed) + return true, "" + } + case http_condNone: + if http_checkIfModifiedSince(r, modtime) == http_condFalse { + http_writeNotModified(w) + return true, "" + } + } + + rangeHeader = r.Header.get("Range") + if rangeHeader != "" && http_checkIfRange(w, r, modtime) == http_condFalse { + rangeHeader = "" + } + return false, rangeHeader } // name is '/'-separated, not filepath.Separator. -func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) { +func http_serveFile(w http_ResponseWriter, r *http_Request, fs http_FileSystem, name string, redirect bool) { const indexPage = "/index.html" + // redirect .../index.html to .../ + // can't use Redirect() because that would make the path absolute, + // which would be a problem running under StripPrefix if strings.HasSuffix(r.URL.Path, indexPage) { - localRedirect(w, r, "./") + http_localRedirect(w, r, "./") return } f, err := fs.Open(name) if err != nil { - msg, code := toHTTPError(err) - Error(w, msg, code) + msg, code := http_toHTTPError(err) + http_serveError(w, msg, code) return } defer f.Close() - d, err1 := f.Stat() - if err1 != nil { - msg, code := toHTTPError(err) - Error(w, msg, code) + d, err := f.Stat() + if err != nil { + msg, code := http_toHTTPError(err) + http_serveError(w, msg, code) return } if redirect { - + // redirect to canonical path: / at end of directory url + // r.URL.Path always begins with / url := r.URL.Path if d.IsDir() { if url[len(url)-1] != '/' { - localRedirect(w, r, path.Base(url)+"/") + http_localRedirect(w, r, path.Base(url)+"/") return } - } else { - if url[len(url)-1] == '/' { - localRedirect(w, r, "../"+path.Base(url)) + } else if url[len(url)-1] == '/' { + base := path.Base(url) + if base == "/" || base == "." { + // The FileSystem maps a path like "/" or "/./" to a file instead of a directory. + msg := "http: attempting to traverse a non-directory" + http_serveError(w, msg, http_StatusInternalServerError) return } + http_localRedirect(w, r, "../"+base) + return } } if d.IsDir() { + url := r.URL.Path + // redirect if the directory name doesn't end in a slash + if url == "" || url[len(url)-1] != '/' { + http_localRedirect(w, r, path.Base(url)+"/") + return + } + + // use contents of index.html for directory, if present index := strings.TrimSuffix(name, "/") + indexPage ff, err := fs.Open(index) if err == nil { defer ff.Close() dd, err := ff.Stat() if err == nil { - name = index d = dd f = ff } } } + // Still a directory? (we didn't find an index.html file) if d.IsDir() { - if checkLastModified(w, r, d.ModTime()) { + if http_checkIfModifiedSince(r, d.ModTime()) == http_condFalse { + http_writeNotModified(w) return } - dirList(w, f) + http_setLastModified(w, d.ModTime()) + http_dirList(w, r, f) return } + // serveContent will check modification time sizeFunc := func() (int64, error) { return d.Size(), nil } - serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) + http_serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) } // toHTTPError returns a non-specific HTTP error message and status code @@ -1449,108 +3396,293 @@ // actually return err.Error(), since msg and httpStatus are returned to users, // and historically Go's ServeContent always returned just "404 Not Found" for // all errors. We don't want to start leaking information in error messages. -func toHTTPError(err error) (msg string, httpStatus int) { - if os.IsNotExist(err) { - return "404 page not found", StatusNotFound +func http_toHTTPError(err error) (msg string, httpStatus int) { + if errors.Is(err, fs.ErrNotExist) { + return "404 page not found", http_StatusNotFound } - if os.IsPermission(err) { - return "403 Forbidden", StatusForbidden + if errors.Is(err, fs.ErrPermission) { + return "403 Forbidden", http_StatusForbidden } - - return "500 Internal Server Error", StatusInternalServerError + if errors.Is(err, http_errInvalidUnsafePath) { + return "404 page not found", http_StatusNotFound + } + // Default: + return "500 Internal Server Error", http_StatusInternalServerError } // localRedirect gives a Moved Permanently response. // It does not convert relative paths to absolute paths like Redirect does. -func localRedirect(w ResponseWriter, r *Request, newPath string) { +func http_localRedirect(w http_ResponseWriter, r *http_Request, newPath string) { + // There is no reliable way for us to redirect correctly when the path has + // escaped slashes, since StripPrefix might be in use. Just return 404. + if p := r.URL.EscapedPath(); strings.Contains(p, "%2f") || strings.Contains(p, "%2F") { + http_NotFound(w, r) + return + } if q := r.URL.RawQuery; q != "" { newPath += "?" + q } w.Header().Set("Location", newPath) - w.WriteHeader(StatusMovedPermanently) + w.WriteHeader(http_StatusMovedPermanently) } // ServeFile replies to the request with the contents of the named // file or directory. // -// As a special case, ServeFile redirects any request where r.URL.Path +// If the provided file or directory name is a relative path, it is +// interpreted relative to the current directory and may ascend to +// parent directories. If the provided name is constructed from user +// input, it should be sanitized before calling [ServeFile]. +// +// As a precaution, ServeFile will reject requests where r.URL.Path +// contains a ".." path element; this protects against callers who +// might unsafely use [filepath.Join] on r.URL.Path without sanitizing +// it and then use that filepath.Join result as the name argument. +// +// As another special case, ServeFile redirects any request where r.URL.Path // ends in "/index.html" to the same path, without the final // "index.html". To avoid such redirects either modify the path or -// use ServeContent. -func ServeFile(w ResponseWriter, r *Request, name string) { +// use [ServeContent]. +// +// Outside of those two special cases, ServeFile does not use +// r.URL.Path for selecting the file or directory to serve; only the +// file or directory provided in the name argument is used. +func http_ServeFile(w http_ResponseWriter, r *http_Request, name string) { + if http_containsDotDot(r.URL.Path) { + // Too many programs use r.URL.Path to construct the argument to + // serveFile. Reject the request under the assumption that happened + // here and ".." may not be wanted. + // Note that name might not contain "..", for example if code (still + // incorrectly) used filepath.Join(myDir, r.URL.Path). + http_serveError(w, "invalid URL path", http_StatusBadRequest) + return + } dir, file := filepath.Split(name) - serveFile(w, r, Dir(dir), file, false) + http_serveFile(w, r, http_Dir(dir), file, false) } -type fileHandler struct { - root FileSystem +// ServeFileFS replies to the request with the contents +// of the named file or directory from the file system fsys. +// The files provided by fsys must implement [io.Seeker]. +// +// If the provided name is constructed from user input, it should be +// sanitized before calling [ServeFileFS]. +// +// As a precaution, ServeFileFS will reject requests where r.URL.Path +// contains a ".." path element; this protects against callers who +// might unsafely use [filepath.Join] on r.URL.Path without sanitizing +// it and then use that filepath.Join result as the name argument. +// +// As another special case, ServeFileFS redirects any request where r.URL.Path +// ends in "/index.html" to the same path, without the final +// "index.html". To avoid such redirects either modify the path or +// use [ServeContent]. +// +// Outside of those two special cases, ServeFileFS does not use +// r.URL.Path for selecting the file or directory to serve; only the +// file or directory provided in the name argument is used. +func http_ServeFileFS(w http_ResponseWriter, r *http_Request, fsys fs.FS, name string) { + if http_containsDotDot(r.URL.Path) { + // Too many programs use r.URL.Path to construct the argument to + // serveFile. Reject the request under the assumption that happened + // here and ".." may not be wanted. + // Note that name might not contain "..", for example if code (still + // incorrectly) used filepath.Join(myDir, r.URL.Path). + http_serveError(w, "invalid URL path", http_StatusBadRequest) + return + } + http_serveFile(w, r, http_FS(fsys), name, false) +} + +func http_containsDotDot(v string) bool { + if !strings.Contains(v, "..") { + return false + } + for ent := range strings.FieldsFuncSeq(v, http_isSlashRune) { + if ent == ".." { + return true + } + } + return false +} + +func http_isSlashRune(r rune) bool { return r == '/' || r == '\\' } + +type http_fileHandler struct { + root http_FileSystem +} + +type http_ioFS struct { + fsys fs.FS +} + +type http_ioFile struct { + file fs.File +} + +func (f http_ioFS) Open(name string) (http_File, error) { + if name == "/" { + name = "." + } else { + name = strings.TrimPrefix(name, "/") + } + file, err := f.fsys.Open(name) + if err != nil { + return nil, http_mapOpenError(err, name, '/', func(path string) (fs.FileInfo, error) { + return fs.Stat(f.fsys, path) + }) + } + return http_ioFile{file}, nil +} + +func (f http_ioFile) Close() error { return f.file.Close() } + +func (f http_ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } + +func (f http_ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } + +var http_errMissingSeek = errors.New("io.File missing Seek method") + +var http_errMissingReadDir = errors.New("io.File directory missing ReadDir method") + +func (f http_ioFile) Seek(offset int64, whence int) (int64, error) { + s, ok := f.file.(io.Seeker) + if !ok { + return 0, http_errMissingSeek + } + return s.Seek(offset, whence) +} + +func (f http_ioFile) ReadDir(count int) ([]fs.DirEntry, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, http_errMissingReadDir + } + return d.ReadDir(count) +} + +func (f http_ioFile) Readdir(count int) ([]fs.FileInfo, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, http_errMissingReadDir + } + var list []fs.FileInfo + for { + dirs, err := d.ReadDir(count - len(list)) + for _, dir := range dirs { + info, err := dir.Info() + if err != nil { + // Pretend it doesn't exist, like (*os.File).Readdir does. + continue + } + list = append(list, info) + } + if err != nil { + return list, err + } + if count < 0 || len(list) >= count { + break + } + } + return list, nil +} + +// FS converts fsys to a [FileSystem] implementation, +// for use with [FileServer] and [NewFileTransport]. +// The files provided by fsys must implement [io.Seeker]. +func http_FS(fsys fs.FS) http_FileSystem { + return http_ioFS{fsys} } // FileServer returns a handler that serves HTTP requests // with the contents of the file system rooted at root. // -// To use the operating system's file system implementation, -// use http.Dir: +// As a special case, the returned file server redirects any request +// ending in "/index.html" to the same path, without the final +// "index.html". // -// http.Handle("/", http.FileServer(http.Dir("/tmp"))) +// To use the operating system's file system implementation, +// use [http.Dir]: +// +// http.Handle("/", http.FileServer(http.Dir("/tmp"))) +// +// To use an [fs.FS] implementation, use [http.FileServerFS] instead. +func http_FileServer(root http_FileSystem) http_Handler { + return &http_fileHandler{root} +} + +// FileServerFS returns a handler that serves HTTP requests +// with the contents of the file system fsys. +// The files provided by fsys must implement [io.Seeker]. // // As a special case, the returned file server redirects any request // ending in "/index.html" to the same path, without the final // "index.html". -func FileServer(root FileSystem) Handler { - return &fileHandler{root} +// +// http.Handle("/", http.FileServerFS(fsys)) +func http_FileServerFS(root fs.FS) http_Handler { + return http_FileServer(http_FS(root)) } -func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { +func (f *http_fileHandler) ServeHTTP(w http_ResponseWriter, r *http_Request) { upath := r.URL.Path if !strings.HasPrefix(upath, "/") { upath = "/" + upath r.URL.Path = upath } - serveFile(w, r, f.root, path.Clean(upath), true) + http_serveFile(w, r, f.root, path.Clean(upath), true) } // httpRange specifies the byte range to be sent to the client. -type httpRange struct { +type http_httpRange struct { start, length int64 } -func (r httpRange) contentRange(size int64) string { +func (r http_httpRange) contentRange(size int64) string { return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) } -func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { +func (r http_httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { return textproto.MIMEHeader{ "Content-Range": {r.contentRange(size)}, "Content-Type": {contentType}, } } -// parseRange parses a Range header string as per RFC 2616. -func parseRange(s string, size int64) ([]httpRange, error) { +// parseRange parses a Range header string as per RFC 7233. +// errNoOverlap is returned if none of the ranges overlap. +func http_parseRange(s string, size int64) ([]http_httpRange, error) { if s == "" { - return nil, nil + return nil, nil // header not present } const b = "bytes=" if !strings.HasPrefix(s, b) { return nil, errors.New("invalid range") } - var ranges []httpRange - for _, ra := range strings.Split(s[len(b):], ",") { - ra = strings.TrimSpace(ra) + var ranges []http_httpRange + noOverlap := false + for ra := range strings.SplitSeq(s[len(b):], ",") { + ra = textproto.TrimString(ra) if ra == "" { continue } - i := strings.Index(ra, "-") - if i < 0 { + start, end, ok := strings.Cut(ra, "-") + if !ok { return nil, errors.New("invalid range") } - start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) - var r httpRange + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r http_httpRange if start == "" { - + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with <suffix-length> + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } i, err := strconv.ParseInt(end, 10, 64) - if err != nil { + if i < 0 || err != nil { return nil, errors.New("invalid range") } if i > size { @@ -1560,12 +3692,18 @@ r.length = size - r.start } else { i, err := strconv.ParseInt(start, 10, 64) - if err != nil || i >= size || i < 0 { + if err != nil || i < 0 { return nil, errors.New("invalid range") } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } r.start = i if end == "" { - + // If no end is specified, range extends to end of the file. r.length = size - r.start } else { i, err := strconv.ParseInt(end, 10, 64) @@ -1580,21 +3718,25 @@ } ranges = append(ranges, r) } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, http_errNoOverlap + } return ranges, nil } // countingWriter counts how many bytes have been written to it. -type countingWriter int64 +type http_countingWriter int64 -func (w *countingWriter) Write(p []byte) (n int, err error) { - *w += countingWriter(len(p)) +func (w *http_countingWriter) Write(p []byte) (n int, err error) { + *w += http_countingWriter(len(p)) return len(p), nil } // rangesMIMESize returns the number of bytes it takes to encode the // provided ranges as a multipart response. -func rangesMIMESize(ranges []httpRange, contentType string, contentSize int64) (encSize int64) { - var w countingWriter +func http_rangesMIMESize(ranges []http_httpRange, contentType string, contentSize int64) (encSize int64) { + var w http_countingWriter mw := multipart.NewWriter(&w) for _, ra := range ranges { mw.CreatePart(ra.mimeHeader(contentType, contentSize)) @@ -1605,78 +3747,124 @@ return } -func sumRangesSize(ranges []httpRange) (size int64) { +func http_sumRangesSize(ranges []http_httpRange) (size int64) { for _, ra := range ranges { size += ra.length } return } -var raceEnabled = false // set by race.go - // A Header represents the key-value pairs in an HTTP header. -type Header map[string][]string +// +// The keys should be in canonical form, as returned by +// [CanonicalHeaderKey]. +type http_Header map[string][]string // Add adds the key, value pair to the header. // It appends to any existing values associated with key. -func (h Header) Add(key, value string) { +// The key is case insensitive; it is canonicalized by +// [CanonicalHeaderKey]. +func (h http_Header) Add(key, value string) { textproto.MIMEHeader(h).Add(key, value) } -// Set sets the header entries associated with key to -// the single element value. It replaces any existing -// values associated with key. -func (h Header) Set(key, value string) { +// Set sets the header entries associated with key to the +// single element value. It replaces any existing values +// associated with key. The key is case insensitive; it is +// canonicalized by [textproto.CanonicalMIMEHeaderKey]. +// To use non-canonical keys, assign to the map directly. +func (h http_Header) Set(key, value string) { textproto.MIMEHeader(h).Set(key, value) } -// Get gets the first value associated with the given key. -// If there are no values associated with the key, Get returns "". -// To access multiple values of a key, access the map directly -// with CanonicalHeaderKey. -func (h Header) Get(key string) string { +// Get gets the first value associated with the given key. If +// there are no values associated with the key, Get returns "". +// It is case insensitive; [textproto.CanonicalMIMEHeaderKey] is +// used to canonicalize the provided key. Get assumes that all +// keys are stored in canonical form. To use non-canonical keys, +// access the map directly. +func (h http_Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } +// Values returns all values associated with the given key. +// It is case insensitive; [textproto.CanonicalMIMEHeaderKey] is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h http_Header) Values(key string) []string { + return textproto.MIMEHeader(h).Values(key) +} + // get is like Get, but key must already be in CanonicalHeaderKey form. -func (h Header) get(key string) string { +func (h http_Header) get(key string) string { if v := h[key]; len(v) > 0 { return v[0] } return "" } +// has reports whether h has the provided key defined, even if it's +// set to 0-length slice. +func (h http_Header) has(key string) bool { + _, ok := h[key] + return ok +} + // Del deletes the values associated with key. -func (h Header) Del(key string) { +// The key is case insensitive; it is canonicalized by +// [CanonicalHeaderKey]. +func (h http_Header) Del(key string) { textproto.MIMEHeader(h).Del(key) } // Write writes a header in wire format. -func (h Header) Write(w io.Writer) error { - return h.WriteSubset(w, nil) +func (h http_Header) Write(w io.Writer) error { + return h.write(w, nil) } -func (h Header) clone() Header { - h2 := make(Header, len(h)) +func (h http_Header) write(w io.Writer, trace *httptrace.ClientTrace) error { + return h.writeSubset(w, nil, trace) +} + +// Clone returns a copy of h or nil if h is nil. +func (h http_Header) Clone() http_Header { + if h == nil { + return nil + } + + // Find total number of values. + nv := 0 + for _, vv := range h { + nv += len(vv) + } + sv := make([]string, nv) // shared backing array for headers' values + h2 := make(http_Header, len(h)) for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 + if vv == nil { + // Preserve nil values. ReverseProxy distinguishes + // between nil and zero-length header values. + h2[k] = nil + continue + } + n := copy(sv, vv) + h2[k] = sv[:n:n] + sv = sv[n:] } return h2 } -var timeFormats = []string{ - TimeFormat, +var http_timeFormats = []string{ + http_TimeFormat, time.RFC850, time.ANSIC, } // ParseTime parses a time header (such as the Date: header), // trying each of the three formats allowed by HTTP/1.1: -// TimeFormat, time.RFC850, and time.ANSIC. -func ParseTime(text string) (t time.Time, err error) { - for _, layout := range timeFormats { +// [TimeFormat], [time.RFC850], and [time.ANSIC]. +func http_ParseTime(text string) (t time.Time, err error) { + for _, layout := range http_timeFormats { t, err = time.Parse(layout, text) if err == nil { return @@ -1685,99 +3873,108 @@ return } -var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") - -type writeStringer interface { - WriteString(string) (int, error) -} +var http_headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") // stringWriter implements WriteString on a Writer. -type stringWriter struct { +type http_stringWriter struct { w io.Writer } -func (w stringWriter) WriteString(s string) (n int, err error) { +func (w http_stringWriter) WriteString(s string) (n int, err error) { return w.w.Write([]byte(s)) } -type keyValues struct { +type http_keyValues struct { key string values []string } -// A headerSorter implements sort.Interface by sorting a []keyValues -// by key. It's used as a pointer, so it can fit in a sort.Interface -// interface value without allocation. -type headerSorter struct { - kvs []keyValues +// headerSorter contains a slice of keyValues sorted by keyValues.key. +type http_headerSorter struct { + kvs []http_keyValues } -func (s *headerSorter) Len() int { return len(s.kvs) } - -func (s *headerSorter) Swap(i, j int) { s.kvs[i], s.kvs[j] = s.kvs[j], s.kvs[i] } - -func (s *headerSorter) Less(i, j int) bool { return s.kvs[i].key < s.kvs[j].key } - -var headerSorterPool = sync.Pool{ - New: func() interface{} { return new(headerSorter) }, +var http_headerSorterPool = sync.Pool{ + New: func() any { return new(http_headerSorter) }, } // sortedKeyValues returns h's keys sorted in the returned kvs // slice. The headerSorter used to sort is also returned, for possible // return to headerSorterCache. -func (h Header) sortedKeyValues(exclude map[string]bool) (kvs []keyValues, hs *headerSorter) { - hs = headerSorterPool.Get().(*headerSorter) +func (h http_Header) sortedKeyValues(exclude map[string]bool) (kvs []http_keyValues, hs *http_headerSorter) { + hs = http_headerSorterPool.Get().(*http_headerSorter) if cap(hs.kvs) < len(h) { - hs.kvs = make([]keyValues, 0, len(h)) + hs.kvs = make([]http_keyValues, 0, len(h)) } kvs = hs.kvs[:0] for k, vv := range h { if !exclude[k] { - kvs = append(kvs, keyValues{k, vv}) + kvs = append(kvs, http_keyValues{k, vv}) } } hs.kvs = kvs - sort.Sort(hs) + slices.SortFunc(hs.kvs, func(a, b http_keyValues) int { return strings.Compare(a.key, b.key) }) return kvs, hs } // WriteSubset writes a header in wire format. // If exclude is not nil, keys where exclude[key] == true are not written. -func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { - ws, ok := w.(writeStringer) +// Keys are not canonicalized before checking the exclude map. +func (h http_Header) WriteSubset(w io.Writer, exclude map[string]bool) error { + return h.writeSubset(w, exclude, nil) +} + +func (h http_Header) writeSubset(w io.Writer, exclude map[string]bool, trace *httptrace.ClientTrace) error { + ws, ok := w.(io.StringWriter) if !ok { - ws = stringWriter{w} + ws = http_stringWriter{w} } kvs, sorter := h.sortedKeyValues(exclude) + var formattedVals []string for _, kv := range kvs { + if !httpguts.ValidHeaderFieldName(kv.key) { + // This could be an error. In the common case of + // writing response headers, however, we have no good + // way to provide the error back to the server + // handler, so just drop invalid headers instead. + continue + } for _, v := range kv.values { - v = headerNewlineToSpace.Replace(v) + v = http_headerNewlineToSpace.Replace(v) v = textproto.TrimString(v) for _, s := range []string{kv.key, ": ", v, "\r\n"} { if _, err := ws.WriteString(s); err != nil { + http_headerSorterPool.Put(sorter) return err } } + if trace != nil && trace.WroteHeaderField != nil { + formattedVals = append(formattedVals, v) + } + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(kv.key, formattedVals) + formattedVals = nil } } - headerSorterPool.Put(sorter) + http_headerSorterPool.Put(sorter) return nil } // CanonicalHeaderKey returns the canonical format of the -// header key s. The canonicalization converts the first +// header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the +// the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // If s contains a space or invalid header field bytes, it is // returned without modifications. -func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } +func http_CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } // hasToken reports whether token appears with v, ASCII // case-insensitive, with space or comma boundaries. // token must be all lowercase. // v may contain mixed cased. -func hasToken(v, token string) bool { +func http_hasToken(v, token string) bool { if len(token) > len(v) || token == "" { return false } @@ -1785,236 +3982,1578 @@ return true } for sp := 0; sp <= len(v)-len(token); sp++ { - + // Check that first character is good. + // The token is ASCII, so checking only a single byte + // is sufficient. We skip this potential starting + // position if both the first byte and its potential + // ASCII uppercase equivalent (b|0x20) don't match. + // False positives ('^' => '~') are caught by EqualFold. if b := v[sp]; b != token[0] && b|0x20 != token[0] { continue } - - if sp > 0 && !isTokenBoundary(v[sp-1]) { + // Check that start pos is on a valid token boundary. + if sp > 0 && !http_isTokenBoundary(v[sp-1]) { continue } - - if endPos := sp + len(token); endPos != len(v) && !isTokenBoundary(v[endPos]) { + // Check that end pos is on a valid token boundary. + if endPos := sp + len(token); endPos != len(v) && !http_isTokenBoundary(v[endPos]) { continue } - if strings.EqualFold(v[sp:sp+len(token)], token) { + if ascii.EqualFold(v[sp:sp+len(token)], token) { return true } } return false } -func isTokenBoundary(b byte) bool { +func http_isTokenBoundary(b byte) bool { return b == ' ' || b == ',' || b == '\t' } +// Protocols is a set of HTTP protocols. +// The zero value is an empty set of protocols. +// +// The supported protocols are: +// +// - HTTP1 is the HTTP/1.0 and HTTP/1.1 protocols. +// HTTP1 is supported on both unsecured TCP and secured TLS connections. +// +// - HTTP2 is the HTTP/2 protocol over a TLS connection. +// +// - UnencryptedHTTP2 is the HTTP/2 protocol over an unsecured TCP connection. +type http_Protocols struct { + bits uint8 +} + +const ( + http_protoHTTP1 = 1 << iota + http_protoHTTP2 + http_protoUnencryptedHTTP2 + http_protoHTTP3 +) + +// HTTP1 reports whether p includes HTTP/1. +func (p http_Protocols) HTTP1() bool { return p.bits&http_protoHTTP1 != 0 } + +// SetHTTP1 adds or removes HTTP/1 from p. +func (p *http_Protocols) SetHTTP1(ok bool) { p.setBit(http_protoHTTP1, ok) } + +// HTTP2 reports whether p includes HTTP/2. +func (p http_Protocols) HTTP2() bool { return p.bits&http_protoHTTP2 != 0 } + +// SetHTTP2 adds or removes HTTP/2 from p. +func (p *http_Protocols) SetHTTP2(ok bool) { p.setBit(http_protoHTTP2, ok) } + +// UnencryptedHTTP2 reports whether p includes unencrypted HTTP/2. +func (p http_Protocols) UnencryptedHTTP2() bool { return p.bits&http_protoUnencryptedHTTP2 != 0 } + +// SetUnencryptedHTTP2 adds or removes unencrypted HTTP/2 from p. +func (p *http_Protocols) SetUnencryptedHTTP2(ok bool) { p.setBit(http_protoUnencryptedHTTP2, ok) } + +// http3 reports whether p includes HTTP/3. +func (p http_Protocols) http3() bool { return p.bits&http_protoHTTP3 != 0 } + +// setHTTP3 adds or removes HTTP/3 from p. +func (p *http_Protocols) setHTTP3(ok bool) { p.setBit(http_protoHTTP3, ok) } + +//go:linkname protocolSetHTTP3 golang.org/x/net/internal/http3_test.protocolSetHTTP3 +func http_protocolSetHTTP3(p *http_Protocols) { p.setHTTP3(true) } + +func (p *http_Protocols) setBit(bit uint8, ok bool) { + if ok { + p.bits |= bit + } else { + p.bits &^= bit + } +} + +// empty returns true if p has no protocol set at all. +func (p http_Protocols) empty() bool { + return p.bits == 0 +} + +func (p http_Protocols) String() string { + var s []string + if p.HTTP1() { + s = append(s, "HTTP1") + } + if p.HTTP2() { + s = append(s, "HTTP2") + } + if p.UnencryptedHTTP2() { + s = append(s, "UnencryptedHTTP2") + } + if p.http3() { + s = append(s, "HTTP3") + } + return "{" + strings.Join(s, ",") + "}" +} + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type http_incomparable [0]func() + +// maxInt64 is the effective "infinite" value for the Server and +// Transport's byte-limiting readers. +const http_maxInt64 = 1<<63 - 1 + +// aLongTimeAgo is a non-zero time, far in the past, used for +// immediate cancellation of network operations. +var http_aLongTimeAgo = time.Unix(1, 0) + +// omitBundledHTTP2 is set by omithttp2.go when the nethttpomithttp2 +// build tag is set. That means h2_bundle.go isn't compiled in and we +// shouldn't try to use it. +var http_omitBundledHTTP2 bool + +// TODO(bradfitz): move common stuff here. The other files have accumulated +// generic http stuff in random places. + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. +type http_contextKey struct { + name string +} + +func (k *http_contextKey) String() string { return "net/http context value " + k.name } + +// removePort strips the port while correctly handling IPv6. +func http_removePort(host string) string { + for i := len(host) - 1; i >= 0; i-- { + switch host[i] { + case ':': + return host[:i] + case ']': + return host + } + } + return host +} + +// isToken reports whether v is a valid token (https://www.rfc-editor.org/rfc/rfc2616#section-2.2). +func http_isToken(v string) bool { + // For historical reasons, this function is called ValidHeaderFieldName (see issue #67031). + return httpguts.ValidHeaderFieldName(v) +} + +// stringContainsCTLByte reports whether s contains any ASCII control character. +func http_stringContainsCTLByte(s string) bool { + for i := 0; i < len(s); i++ { + b := s[i] + if b < ' ' || b == 0x7f { + return true + } + } + return false +} + +func http_hexEscapeNonASCII(s string) string { + newLen := 0 + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + newLen += 3 + } else { + newLen++ + } + } + if newLen == len(s) { + return s + } + b := make([]byte, 0, newLen) + var pos int + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + if pos < i { + b = append(b, s[pos:i]...) + } + b = append(b, '%') + b = strconv.AppendInt(b, int64(s[i]), 16) + pos = i + 1 + } + } + if pos < len(s) { + b = append(b, s[pos:]...) + } + return string(b) +} + +// NoBody is an [io.ReadCloser] with no bytes. Read always returns EOF +// and Close always returns nil. It can be used in an outgoing client +// request to explicitly signal that a request has zero bytes. +// An alternative, however, is to simply set [Request.Body] to nil. +var http_NoBody = http_noBody{} + +type http_noBody struct{} + +func (http_noBody) Read([]byte) (int, error) { return 0, io.EOF } + +func (http_noBody) Close() error { return nil } + +func (http_noBody) WriteTo(io.Writer) (int64, error) { return 0, nil } + +var ( + // verify that an io.Copy from NoBody won't require a buffer: + _ io.WriterTo = http_NoBody + _ io.ReadCloser = http_NoBody +) + +// PushOptions describes options for [Pusher.Push]. +type http_PushOptions struct { + // Method specifies the HTTP method for the promised request. + // If set, it must be "GET" or "HEAD". Empty means "GET". + Method string + + // Header specifies additional promised request headers. This cannot + // include HTTP/2 pseudo header fields like ":path" and ":scheme", + // which will be added automatically. + Header http_Header +} + +// Pusher is the interface implemented by ResponseWriters that support +// HTTP/2 server push. For more background, see +// https://tools.ietf.org/html/rfc7540#section-8.2. +type http_Pusher interface { + // Push initiates an HTTP/2 server push. This constructs a synthetic + // request using the given target and options, serializes that request + // into a PUSH_PROMISE frame, then dispatches that request using the + // server's request handler. If opts is nil, default options are used. + // + // The target must either be an absolute path (like "/path") or an absolute + // URL that contains a valid host and the same scheme as the parent request. + // If the target is a path, it will inherit the scheme and host of the + // parent request. + // + // The HTTP/2 spec disallows recursive pushes and cross-authority pushes. + // Push may or may not detect these invalid pushes; however, invalid + // pushes will be detected and canceled by conforming clients. + // + // Handlers that wish to push URL X should call Push before sending any + // data that may trigger a request for URL X. This avoids a race where the + // client issues requests for X before receiving the PUSH_PROMISE for X. + // + // Push will run in a separate goroutine making the order of arrival + // non-deterministic. Any required synchronization needs to be implemented + // by the caller. + // + // Push returns ErrNotSupported if the client has disabled push or if push + // is not supported on the underlying connection. + Push(target string, opts *http_PushOptions) error +} + +// HTTP2Config defines HTTP/2 configuration parameters common to +// both [Transport] and [Server]. +type http_HTTP2Config struct { + // MaxConcurrentStreams optionally specifies the number of + // concurrent streams that a client may have open at a time. + // If zero, MaxConcurrentStreams defaults to at least 100. + // + // This parameter only applies to Servers. + MaxConcurrentStreams int + + // StrictMaxConcurrentRequests controls whether an HTTP/2 server's + // concurrency limit should be respected across all connections + // to that server. + // If true, new requests sent when a connection's concurrency limit + // has been exceeded will block until an existing request completes. + // If false, an additional connection will be opened if all + // existing connections are at their limit. + // + // This parameter only applies to Transports. + StrictMaxConcurrentRequests bool + + // MaxDecoderHeaderTableSize optionally specifies an upper limit for the + // size of the header compression table used for decoding headers sent + // by the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxDecoderHeaderTableSize int + + // MaxEncoderHeaderTableSize optionally specifies an upper limit for the + // header compression table used for sending headers to the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxEncoderHeaderTableSize int + + // MaxReadFrameSize optionally specifies the largest frame + // this endpoint is willing to read. + // A valid value is between 16KiB and 16MiB, inclusive. + // If zero or invalid, a default value is used. + MaxReadFrameSize int + + // MaxReceiveBufferPerConnection is the maximum size of the + // flow control window for data received on a connection. + // A valid value is at least 64KiB and less than 4MiB. + // If invalid, a default value is used. + MaxReceiveBufferPerConnection int + + // MaxReceiveBufferPerStream is the maximum size of + // the flow control window for data received on a stream (request). + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxReceiveBufferPerStream int + + // SendPingTimeout is the timeout after which a health check using a ping + // frame will be carried out if no frame is received on a connection. + // If zero, no health check is performed. + SendPingTimeout time.Duration + + // PingTimeout is the timeout after which a connection will be closed + // if a response to a ping is not received. + // If zero, a default of 15 seconds is used. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which a connection will be + // closed if no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + WriteByteTimeout time.Duration + + // PermitProhibitedCipherSuites, if true, permits the use of + // cipher suites prohibited by the HTTP/2 spec. + PermitProhibitedCipherSuites bool + + // CountError, if non-nil, is called on HTTP/2 errors. + // It is intended to increment a metric for monitoring. + // The errType contains only lowercase letters, digits, and underscores + // (a-z, 0-9, _). + CountError func(errType string) +} + +// net/http supports HTTP/2 by default, but this support is removed when +// the nethttpomithttp2 build tag is set. +// +// HTTP/2 support is provided by the net/http/internal/http2 package. +// +// This file (http2.go) connects net/http to the http2 package. +// Since http imports http2, to avoid an import cycle we need to +// translate http package types (e.g., Request) into the equivalent +// http2 package types (e.g., http2.ClientRequest). +// +// The golang.org/x/net/http2 package is the original source of truth for +// the HTTP/2 implementation. At this time, users may still import that +// package and register its implementation on a net/http Transport or Server. +// However, the x/net package is no longer synchronized with std. + +func init() { + // NoBody and LocalAddrContextKey need to have the same value + // in the http and http2 packages. + // + // We can't define these values in net/http/internal, + // because their concrete types are part of the net/http API and + // moving them causes API checker failures. + // Override the http2 package versions at init time instead. + http2.LocalAddrContextKey = http_LocalAddrContextKey + http2.NoBody = http_NoBody +} + +type http_http2Server = http2.Server + +type http_http2Transport = http2.Transport + +func (s *http_Server) configureHTTP2() { + h2srv := &http2.Server{} + + // Historically, we've configured the HTTP/2 idle timeout in this fashion: + // Set once at configuration time. + if s.IdleTimeout != 0 { + s.h2IdleTimeout = s.IdleTimeout + } else { + s.h2IdleTimeout = s.ReadTimeout + } + + if s.TLSConfig == nil { + s.TLSConfig = &tls.Config{} + } + s.nextProtoErr = h2srv.Configure(http_http2ServerConfig{s}, s.TLSConfig) + if s.nextProtoErr != nil { + return + } + + s.RegisterOnShutdown(h2srv.GracefulShutdown) + + if s.TLSNextProto == nil { + s.TLSNextProto = make(map[string]func(*http_Server, *tls.Conn, http_Handler)) + } + // Historically, the presence of a TLSNextProto["h2"] key has been the signal to + // enable/disable HTTP/2 support. Set a value in the map, but we'll never use it. + s.TLSNextProto["h2"] = func(hs *http_Server, c *tls.Conn, h http_Handler) { + c.Close() + } + + s.h2 = h2srv +} + +func (s *http_Server) setHTTP2Config(conf http_http2ExternalServerConfig) { + if s.h2Config != nil { + panic("http: HTTP/2 Server already registered") + } + s.h2Config = conf + s.h2Config.ServeConnFunc(s.serveHTTP2Conn) +} + +func (s *http_Server) serveHTTP2Conn(ctx context.Context, nc net.Conn, h http_Handler, sawClientPreface bool, upgradeReq *http_Request, settings []byte) { + s.setupHTTP2_ServeTLS() + var serverUpgradeReq *http2.ServerRequest + if upgradeReq != nil { + serverUpgradeReq = http_http2ServerRequestFromRequest(upgradeReq) + } + s.h2.ServeConn(nc, &http2.ServeConnOpts{ + Context: ctx, + Handler: http_http2Handler{h}, + BaseConfig: http_http2ServerConfig{s}, + SawClientPreface: sawClientPreface, + UpgradeRequest: serverUpgradeReq, + Settings: settings, + }) +} + +func http_http2ServerRequestFromRequest(req *http_Request) *http2.ServerRequest { + return &http2.ServerRequest{ + Context: req.Context(), + Proto: req.Proto, + ProtoMajor: req.ProtoMajor, + ProtoMinor: req.ProtoMinor, + Method: req.Method, + URL: req.URL, + Header: http2.Header(req.Header), + Trailer: http2.Header(req.Trailer), + Body: req.Body, + Host: req.Host, + ContentLength: req.ContentLength, + RemoteAddr: req.RemoteAddr, + RequestURI: req.RequestURI, + TLS: req.TLS, + MultipartForm: req.MultipartForm, + } +} + +type http_http2Handler struct { + h http_Handler +} + +func (h http_http2Handler) ServeHTTP(w *http2.ResponseWriter, req *http2.ServerRequest) { + h.h.ServeHTTP(http_http2ResponseWriter{w}, &http_Request{ + ctx: req.Context, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + Method: req.Method, + URL: req.URL, + Header: http_Header(req.Header), + RequestURI: req.RequestURI, + Trailer: http_Header(req.Trailer), + Body: req.Body, + Host: req.Host, + ContentLength: req.ContentLength, + RemoteAddr: req.RemoteAddr, + TLS: req.TLS, + MultipartForm: req.MultipartForm, + }) +} + +type http_http2ResponseWriter struct { + *http2.ResponseWriter +} + +// Optional http.ResponseWriter interfaces implemented. +var ( + _ http_CloseNotifier = http_http2ResponseWriter{} + _ http_Flusher = http_http2ResponseWriter{} + _ io.StringWriter = http_http2ResponseWriter{} +) + +func (w http_http2ResponseWriter) Flush() { w.ResponseWriter.FlushError() } + +func (w http_http2ResponseWriter) FlushError() error { return w.ResponseWriter.FlushError() } + +func (w http_http2ResponseWriter) Header() http_Header { return http_Header(w.ResponseWriter.Header()) } + +func (w http_http2ResponseWriter) Push(target string, opts *http_PushOptions) error { + var ( + method string + header http2.Header + ) + if opts != nil { + method = opts.Method + header = http2.Header(opts.Header) + } + err := w.ResponseWriter.Push(target, method, header) + if err == http2.ErrNotSupported { + err = http_ErrNotSupported + } + return err +} + +type http_http2ServerConfig struct { + s *http_Server +} + +func (s http_http2ServerConfig) MaxHeaderBytes() int { return s.s.MaxHeaderBytes } + +func (s http_http2ServerConfig) MaxHeaderValueCount() int { return s.s.maxHeaderValueCount() } + +func (s http_http2ServerConfig) ConnState(c net.Conn, st http2.ConnState) { + if s.s.ConnState != nil { + s.s.ConnState(c, http_ConnState(st)) + } +} + +func (s http_http2ServerConfig) DoKeepAlives() bool { return s.s.doKeepAlives() } + +func (s http_http2ServerConfig) WriteTimeout() time.Duration { return s.s.WriteTimeout } + +func (s http_http2ServerConfig) SendPingTimeout() time.Duration { return s.s.ReadTimeout } + +func (s http_http2ServerConfig) ErrorLog() *log.Logger { return s.s.ErrorLog } + +func (s http_http2ServerConfig) ReadTimeout() time.Duration { return s.s.ReadTimeout } + +func (s http_http2ServerConfig) DisableClientPriority() bool { return s.s.DisableClientPriority } + +func (s http_http2ServerConfig) IdleTimeout() time.Duration { + if s.s.h2Config != nil { + return s.s.h2Config.IdleTimeout() + } + return s.s.h2IdleTimeout +} + +func (s http_http2ServerConfig) HTTP2Config() http2.Config { + return http_mergeHTTP2Config(s.s.HTTP2, s.s.h2Config) +} + +// http2ExternalServerConfig is an HTTP/2 configuration provided by x/net/http2. +// +// When a x/net/http2.Server wraps a net/http.Server, we need to support the user +// setting configuration settings on the x/net Server: +// +// s1 := &http.Server{} +// s2 := &http2.Server{} +// http2.ConfigureServer(s1, s2) +// +// // This setting needs to affect s1: +// s2.MaxReadFrameSize = 10000 +// +// We handle this by having http2.ConfigureServer pass us an http2ExternalServerConfig +// (see http.Server.Serve) which we can use to query the current state of the http2.Server. +type http_http2ExternalServerConfig interface { + // Various configuration settings: + HTTP2Config() http_HTTP2Config + IdleTimeout() time.Duration + + // ServeConnFunc provides a function to the x/net/http2.Server which it + // can use to serve a new connection. + ServeConnFunc(func(ctx context.Context, nc net.Conn, h http_Handler, sawClientPreface bool, upgradeReq *http_Request, settings []byte)) +} + +// http2ExternalTransportConfig is an HTTP/2 configuration provided by x/net/http2. +// +// When a x/net/http2.Transport wraps a net/http.Transport, we need to support the user +// setting configuration settings on the x/net Transport: +// +// tr1 := &http.Transport{} +// tr2 := http2.ConfigureTransports(t1) +// +// // This setting needs to affect tr1: +// tr2.MaxHeaderListSize = 10000 +// +// We handle this by having http2.ConfigureTransports pass us an http2ExternalTransportConfig, +// which we can use to query the current state of the http2.Transport. +type http_http2ExternalTransportConfig interface { + // Various configuration settings: + HTTP2Config() http_HTTP2Config + DisableCompression() bool + MaxHeaderListSize() int64 + IdleConnTimeout() time.Duration + + // ConnFromContext is used to pass a net.Conn to Transport.NewClientConn + // via a context value. See Transport.http2NewClientConnFromContext. + ConnFromContext(context.Context) net.Conn + + // DialFromContext is used to dial new connections, overriding Transport.DialContext etc. + // This is used when the user calls x/net/http2.Transport.RoundTrip directly, + // in which case the historical behavior is to use the http2.Transport's dial functions. + DialFromContext(ctx context.Context, network, addr string) (net.Conn, error) + + // ExternalRoundTrip reports whether Transport.RoundTrip should call the + // external transport's RoundTrip. This is used when x/net/http2.Transport.ConnPool + // is set, in which case the user-provided ClientConnPool has taken responsibility + // for picking a connection to use. + ExternalRoundTrip() bool + + // RoundTrip performs a round trip. + // It should only be used when ExternalRoundTrip requests it. + RoundTrip(*http_Request) (*http_Response, error) + + // Registered is called to report successful registration of the config. + Registered(*http_Transport) +} + +func (t *http_Transport) configureHTTP2(protocols http_Protocols) { + if t.TLSClientConfig == nil { + t.TLSClientConfig = &tls.Config{} + } + if t.HTTP2 == nil { + t.HTTP2 = &http_HTTP2Config{} + } + t2 := http2.NewTransport(http_transportConfig{t}) + t.h2Transport = t2 + + t.registerProtocol("https", http_http2RoundTripper{t2, true}) + if t.TLSNextProto == nil { + t.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http_RoundTripper) + } + // Historically, the presence of a TLSNextProto["h2"] key has been the signal to + // enable/disable HTTP/2 support. Set a value in the map, but we'll never use it. + t.TLSNextProto["h2"] = func(authority string, c *tls.Conn) http_RoundTripper { + return http_http2ErringRoundTripper{ + errors.New("unexpected use of stub RoundTripper"), + } + } + + // Server.ServeTLS clones the tls.Config before modifying it. + // Transport doesn't. We may want to make the two consistent some day. + // + // http2configureTransport will have already set NextProtos, but adjust it again + // here to remove HTTP/1.1 if the user has disabled it. + t.TLSClientConfig.NextProtos = http_adjustNextProtos(t.TLSClientConfig.NextProtos, protocols) +} + +type http_http2ErringRoundTripper struct{ err error } + +func (rt http_http2ErringRoundTripper) RoundTripErr() error { return rt.err } + +func (rt http_http2ErringRoundTripper) RoundTrip(*http_Request) (*http_Response, error) { + return nil, rt.err +} + +func http_http2RoundTrip(req *http_Request, rt func(*http2.ClientRequest) (*http2.ClientResponse, error)) (*http_Response, error) { + resp := &http_Response{} + cresp, err := rt(&http2.ClientRequest{ + Context: req.Context(), + Method: req.Method, + URL: req.URL, + Header: http2.Header(req.Header), + Trailer: http2.Header(req.Trailer), + Body: req.Body, + Host: req.Host, + GetBody: req.GetBody, + ContentLength: req.ContentLength, + Cancel: req.Cancel, + Close: req.Close, + ResTrailer: (*http2.Header)(&resp.Trailer), + }) + if err != nil { + return nil, err + } + resp.Status = cresp.Status + " " + http_StatusText(cresp.StatusCode) + resp.StatusCode = cresp.StatusCode + resp.Proto = "HTTP/2.0" + resp.ProtoMajor = 2 + resp.ProtoMinor = 0 + resp.ContentLength = cresp.ContentLength + resp.Uncompressed = cresp.Uncompressed + resp.Header = http_Header(cresp.Header) + resp.Trailer = http_Header(cresp.Trailer) + resp.Body = cresp.Body + resp.TLS = cresp.TLS + resp.Request = req + return resp, nil +} + +// http2AddConn adds nc to the HTTP/2 connection pool. +func (t *http_Transport) http2AddConn(scheme, authority string, nc net.Conn) (http_RoundTripper, error) { + if t.h2Transport == nil { + return nil, errors.ErrUnsupported + } + err := t.h2Transport.AddConn(scheme, authority, nc) + if err != nil { + return nil, err + } + return http_http2RoundTripper{t.h2Transport, false}, nil +} + +// http2NewClientConn creates an HTTP/2 genericClientConn (used to implement ClientConn) from nc. +// The connection is not added to the HTTP/2 connection pool. +func (t *http_Transport) http2NewClientConn(nc net.Conn, internalStateHook func()) (http_genericClientConn, error) { + if t.h2Transport == nil { + return nil, errors.ErrUnsupported + } + cc, err := t.h2Transport.NewClientConn(nc, internalStateHook) + if err != nil { + return nil, err + } + return http_http2ClientConn{cc}, nil +} + +// http2NewClientConnFromContext creates a *ClientConn from a net.Conn. +// +// Transport.NewClientConn takes an address and dials a new net.Conn. +// We don't currently provide a simple way for the user to provide a net.Conn and get a +// *ClientConn out of it (although we do let the user provide their own Transport.DialContext, +// which can be used to effectively do this). +// +// x/net/http2.Transport.NewClientConn, in contrast, requires the user to provide a net.Conn. +// To support implementing the x/net/http2 NewClientConn in terms of a net/http.Transport, +// we permit x/net/http2 to pass us a net.Conn via a context key. +// +// http2NewClientConnFromContext handles extracting the net.Conn from the Context +// (when present) and creating a *ClientConn from it. +func (t *http_Transport) http2NewClientConnFromContext(ctx context.Context) (*http_ClientConn, error) { + if t.h2Config == nil { + return nil, errors.ErrUnsupported + } + nc := t.h2Config.ConnFromContext(ctx) + if nc == nil { + return nil, errors.ErrUnsupported + } + if t.h2Transport == nil { + return nil, errors.New("http: Transport does not support HTTP/2") + } + cc := &http_ClientConn{} + gc, err := t.http2NewClientConn(nc, cc.maybeRunStateHook) + if err != nil { + return nil, err + } + cc.stateHookMu.Lock() + defer cc.stateHookMu.Unlock() + cc.cc = gc + cc.lastAvailable = gc.Available() + return cc, nil +} + +// http2ExternalDial creates a new HTTP/2 connection, +// using the x/net/http2.Transport's dial functions. +// +// This is used when the user has called x/net/http2.Transport.RoundTrip. +// If the RoundTrip needs to create a new connection, +// the historical behavior is for it to use the http2.Transport's DialTLS or DialTLSContext +// functions, and not any dial functions on the http.Transport. +func (t *http_Transport) http2ExternalDial(ctx context.Context, cm http_connectMethod) (http_RoundTripper, error) { + if t.h2Config == nil { + return nil, errors.ErrUnsupported + } + nc, err := t.h2Config.DialFromContext(ctx, "tcp", cm.targetAddr) + if err != nil { + return nil, err + } + return t.http2AddConn(cm.targetScheme, cm.targetAddr, nc) +} + +type http_http2RoundTripper struct { + t *http2.Transport + mapCachedConnErr bool +} + +func (rt http_http2RoundTripper) RoundTrip(req *http_Request) (*http_Response, error) { + resp, err := http_http2RoundTrip(req, rt.t.RoundTrip) + if err != nil { + if rt.mapCachedConnErr && http_http2isNoCachedConnError(err) { + err = http_ErrSkipAltProtocol + } + return nil, err + } + return resp, nil +} + +type http_http2ClientConn struct { + http2.NetHTTPClientConn +} + +func (cc http_http2ClientConn) RoundTrip(req *http_Request) (*http_Response, error) { + return http_http2RoundTrip(req, cc.NetHTTPClientConn.RoundTrip) +} + +// transportConfig implements the http2.TransportConfig interface, +// providing the net/http Transport's configuration to the HTTP/2 implementation. +// +// When an x/net/http2 Transport has provided a configuration (see http2ExternalTransportConfig), +// the transportConfig merges the x/net/http2 and net/http Transport configurations. +type http_transportConfig struct { + t *http_Transport +} + +func (t http_transportConfig) MaxResponseHeaderBytes() int64 { return t.t.MaxResponseHeaderBytes } + +func (t http_transportConfig) DisableKeepAlives() bool { return t.t.DisableKeepAlives } + +func (t http_transportConfig) ExpectContinueTimeout() time.Duration { return t.t.ExpectContinueTimeout } + +func (t http_transportConfig) ResponseHeaderTimeout() time.Duration { return t.t.ResponseHeaderTimeout } + +func (t http_transportConfig) MaxHeaderListSize() int64 { + if t.t.h2Config != nil { + return t.t.h2Config.MaxHeaderListSize() + } + return 0 +} + +func (t http_transportConfig) DisableCompression() bool { + if t.t.h2Config != nil && t.t.h2Config.DisableCompression() { + return true + } + return t.t.DisableCompression +} + +func (t http_transportConfig) IdleConnTimeout() time.Duration { + // Unlike most config settings, historically IdleConnTimeout prefers the + // http2.Transport's setting over the http.Transport. + if t.t.h2Config != nil { + if timeout := t.t.h2Config.IdleConnTimeout(); timeout != 0 { + return timeout + } + } + return t.t.IdleConnTimeout +} + +type http_http2Configer interface { + HTTP2Config() http_HTTP2Config +} + +func http_mergeHTTP2Config(c1 *http_HTTP2Config, confer http_http2Configer) http2.Config { + if c1 == nil && confer == nil { + return http2.Config{} + } + var c http2.Config + if c1 != nil { + c = (http2.Config)(*c1) + } + var c2 http_HTTP2Config + if confer != nil { + c2 = confer.HTTP2Config() + } + if c.MaxConcurrentStreams == 0 { + c.MaxConcurrentStreams = c2.MaxConcurrentStreams + } + if c2.StrictMaxConcurrentRequests { + c.StrictMaxConcurrentRequests = true + } + if c.MaxDecoderHeaderTableSize == 0 { + c.MaxDecoderHeaderTableSize = c2.MaxDecoderHeaderTableSize + } + if c.MaxEncoderHeaderTableSize == 0 { + c.MaxEncoderHeaderTableSize = c2.MaxEncoderHeaderTableSize + } + if c.MaxReadFrameSize == 0 { + c.MaxReadFrameSize = c2.MaxReadFrameSize + } + if c.MaxReceiveBufferPerConnection == 0 { + c.MaxReceiveBufferPerConnection = c2.MaxReceiveBufferPerConnection + } + if c.MaxReceiveBufferPerStream == 0 { + c.MaxReceiveBufferPerStream = c2.MaxReceiveBufferPerStream + } + if c.SendPingTimeout == 0 { + c.SendPingTimeout = c2.SendPingTimeout + } + if c.PingTimeout == 0 { + c.PingTimeout = c2.PingTimeout + } + if c.WriteByteTimeout == 0 { + c.WriteByteTimeout = c2.WriteByteTimeout + } + if c2.PermitProhibitedCipherSuites { + c.PermitProhibitedCipherSuites = true + } + if c.CountError == nil { + c.CountError = c2.CountError + } + return c +} + +func (t http_transportConfig) HTTP2Config() http2.Config { + return http_mergeHTTP2Config(t.t.HTTP2, t.t.h2Config) +} + +// transportFromH1Transport provides a way for HTTP/2 tests to extract +// the http2.Transport from an http.Transport. +// +//go:linkname transportFromH1Transport net/http/internal/http2_test.transportFromH1Transport +func http_transportFromH1Transport(t *http_Transport) any { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + return t.h2Transport +} + // A CookieJar manages storage and use of cookies in HTTP requests. // // Implementations of CookieJar must be safe for concurrent use by multiple // goroutines. // // The net/http/cookiejar package provides a CookieJar implementation. -type CookieJar interface { +type http_CookieJar interface { // SetCookies handles the receipt of the cookies in a reply for the // given URL. It may or may not choose to save the cookies, depending // on the jar's policy and implementation. - SetCookies(u *url.URL, cookies []*Cookie) + SetCookies(u *url.URL, cookies []*http_Cookie) // Cookies returns the cookies to send in a request for the given URL. // It is up to the implementation to honor the standard cookie use // restrictions such as in RFC 6265. - Cookies(u *url.URL) []*Cookie + Cookies(u *url.URL) []*http_Cookie } -var isTokenTable = [127]bool{ - '!': true, - '#': true, - '$': true, - '%': true, - '&': true, - '\'': true, - '*': true, - '+': true, - '-': true, - '.': true, - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - 'A': true, - 'B': true, - 'C': true, - 'D': true, - 'E': true, - 'F': true, - 'G': true, - 'H': true, - 'I': true, - 'J': true, - 'K': true, - 'L': true, - 'M': true, - 'N': true, - 'O': true, - 'P': true, - 'Q': true, - 'R': true, - 'S': true, - 'T': true, - 'U': true, - 'W': true, - 'V': true, - 'X': true, - 'Y': true, - 'Z': true, - '^': true, - '_': true, - '` + "`" + `': true, - 'a': true, - 'b': true, - 'c': true, - 'd': true, - 'e': true, - 'f': true, - 'g': true, - 'h': true, - 'i': true, - 'j': true, - 'k': true, - 'l': true, - 'm': true, - 'n': true, - 'o': true, - 'p': true, - 'q': true, - 'r': true, - 's': true, - 't': true, - 'u': true, - 'v': true, - 'w': true, - 'x': true, - 'y': true, - 'z': true, - '|': true, - '~': true, +// A mapping is a collection of key-value pairs where the keys are unique. +// A zero mapping is empty and ready to use. +// A mapping tries to pick a representation that makes [mapping.find] most efficient. +type http_mapping[K comparable, V any] struct { + s []http_entry[K, V] // for few pairs + m map[K]V // for many pairs } -func isToken(r rune) bool { - i := int(r) - return i < len(isTokenTable) && isTokenTable[i] +type http_entry[K comparable, V any] struct { + key K + value V } -func isNotToken(r rune) bool { - return !isToken(r) +// maxSlice is the maximum number of pairs for which a slice is used. +// It is a variable for benchmarking. +var http_maxSlice int = 8 + +// add adds a key-value pair to the mapping. +func (h *http_mapping[K, V]) add(k K, v V) { + if h.m == nil && len(h.s) < http_maxSlice { + h.s = append(h.s, http_entry[K, V]{k, v}) + } else { + if h.m == nil { + h.m = map[K]V{} + for _, e := range h.s { + h.m[e.key] = e.value + } + h.s = nil + } + h.m[k] = v + } } -// headerValuesContainsToken reports whether any string in values -// contains the provided token, ASCII case-insensitively. -func headerValuesContainsToken(values []string, token string) bool { - for _, v := range values { - if headerValueContainsToken(v, token) { - return true +// find returns the value corresponding to the given key. +// The second return value is false if there is no value +// with that key. +func (h *http_mapping[K, V]) find(k K) (v V, found bool) { + if h == nil { + return v, false + } + if h.m != nil { + v, found = h.m[k] + return v, found + } + for _, e := range h.s { + if e.key == k { + return e.value, true } } - return false + return v, false } -// isOWS reports whether b is an optional whitespace byte, as defined -// by RFC 7230 section 3.2.3. -func isOWS(b byte) bool { return b == ' ' || b == '\t' } - -// trimOWS returns x with all optional whitespace removes from the -// beginning and end. -func trimOWS(x string) string { - - for len(x) > 0 && isOWS(x[0]) { - x = x[1:] +// eachPair calls f for each pair in the mapping. +// If f returns false, pairs returns immediately. +func (h *http_mapping[K, V]) eachPair(f func(k K, v V) bool) { + if h == nil { + return } - for len(x) > 0 && isOWS(x[len(x)-1]) { - x = x[:len(x)-1] + if h.m != nil { + for k, v := range h.m { + if !f(k, v) { + return + } + } + } else { + for _, e := range h.s { + if !f(e.key, e.value) { + return + } + } } - return x } -// headerValueContainsToken reports whether v (assumed to be a -// 0#element, in the ABNF extension described in RFC 7230 section 7) -// contains token amongst its comma-separated tokens, ASCII -// case-insensitively. -func headerValueContainsToken(v string, token string) bool { - v = trimOWS(v) - if comma := strings.IndexByte(v, ','); comma != -1 { - return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token) - } - return tokenEqual(v, token) +// Common HTTP methods. +// +// Unless otherwise noted, these are defined in RFC 7231 section 4.3. +const ( + http_MethodGet = "GET" + http_MethodHead = "HEAD" + http_MethodPost = "POST" + http_MethodPut = "PUT" + http_MethodPatch = "PATCH" // RFC 5789 + http_MethodDelete = "DELETE" + http_MethodConnect = "CONNECT" + http_MethodOptions = "OPTIONS" + http_MethodTrace = "TRACE" +) + +// A pattern is something that can be matched against an HTTP request. +// It has an optional method, an optional host, and a path. +type http_pattern struct { + str string // original string + method string + host string + // The representation of a path differs from the surface syntax, which + // simplifies most algorithms. + // + // Paths ending in '/' are represented with an anonymous "..." wildcard. + // For example, the path "a/" is represented as a literal segment "a" followed + // by a segment with multi==true. + // + // Paths ending in "{$}" are represented with the literal segment "/". + // For example, the path "a/{$}" is represented as a literal segment "a" followed + // by a literal segment "/". + segments []http_segment + loc string // source location of registering call, for helpful messages } -// lowerASCII returns the ASCII lowercase version of b. -func lowerASCII(b byte) byte { - if 'A' <= b && b <= 'Z' { - return b + ('a' - 'A') - } - return b +func (p *http_pattern) String() string { return p.str } + +func (p *http_pattern) lastSegment() http_segment { + return p.segments[len(p.segments)-1] } -// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively. -func tokenEqual(t1, t2 string) bool { - if len(t1) != len(t2) { +// A segment is a pattern piece that matches one or more path segments, or +// a trailing slash. +// +// If wild is false, it matches a literal segment, or, if s == "/", a trailing slash. +// Examples: +// +// "a" => segment{s: "a"} +// "/{$}" => segment{s: "/"} +// +// If wild is true and multi is false, it matches a single path segment. +// Example: +// +// "{x}" => segment{s: "x", wild: true} +// +// If both wild and multi are true, it matches all remaining path segments. +// Example: +// +// "{rest...}" => segment{s: "rest", wild: true, multi: true} +type http_segment struct { + s string // literal or wildcard name or "/" for "/{$}". + wild bool + multi bool // "..." wildcard +} + +// parsePattern parses a string into a Pattern. +// The string's syntax is +// +// [METHOD] [HOST]/[PATH] +// +// where: +// - METHOD is an HTTP method +// - HOST is a hostname +// - PATH consists of slash-separated segments, where each segment is either +// a literal or a wildcard of the form "{name}", "{name...}", or "{$}". +// +// METHOD, HOST and PATH are all optional; that is, the string can be "/". +// If METHOD is present, it must be followed by at least one space or tab. +// Wildcard names must be valid Go identifiers. +// The "{$}" and "{name...}" wildcard must occur at the end of PATH. +// PATH may end with a '/'. +// Wildcard names in a path must be distinct. +func http_parsePattern(s string) (_ *http_pattern, err error) { + if len(s) == 0 { + return nil, errors.New("empty pattern") + } + off := 0 // offset into string + defer func() { + if err != nil { + err = fmt.Errorf("at offset %d: %w", off, err) + } + }() + + method, rest, found := s, "", false + if i := strings.IndexAny(s, " \t"); i >= 0 { + method, rest, found = s[:i], strings.TrimLeft(s[i+1:], " \t"), true + } + if !found { + rest = method + method = "" + } + if method != "" && !http_validMethod(method) { + return nil, fmt.Errorf("invalid method %q", method) + } + p := &http_pattern{str: s, method: method} + + if found { + off = len(method) + 1 + } + i := strings.IndexByte(rest, '/') + if i < 0 { + return nil, errors.New("host/path missing /") + } + p.host = rest[:i] + rest = rest[i:] + if j := strings.IndexByte(p.host, '{'); j >= 0 { + off += j + return nil, errors.New("host contains '{' (missing initial '/'?)") + } + // At this point, rest is the path. + off += i + + // An unclean path with a method that is not CONNECT can never match, + // because paths are cleaned before matching. + if method != "" && method != "CONNECT" && rest != http_cleanPath(rest) { + return nil, errors.New("non-CONNECT pattern with unclean path can never match") + } + + seenNames := map[string]bool{} // remember wildcard names to catch dups + for len(rest) > 0 { + // Invariant: rest[0] == '/'. + rest = rest[1:] + off = len(s) - len(rest) + if len(rest) == 0 { + // Trailing slash. + p.segments = append(p.segments, http_segment{wild: true, multi: true}) + break + } + i := strings.IndexByte(rest, '/') + if i < 0 { + i = len(rest) + } + var seg string + seg, rest = rest[:i], rest[i:] + if i := strings.IndexByte(seg, '{'); i < 0 { + // Literal. + seg = http_pathUnescape(seg) + p.segments = append(p.segments, http_segment{s: seg}) + } else { + // Wildcard. + if i != 0 { + return nil, errors.New("bad wildcard segment (must start with '{')") + } + if seg[len(seg)-1] != '}' { + return nil, errors.New("bad wildcard segment (must end with '}')") + } + name := seg[1 : len(seg)-1] + if name == "$" { + if len(rest) != 0 { + return nil, errors.New("{$} not at end") + } + p.segments = append(p.segments, http_segment{s: "/"}) + break + } + name, multi := strings.CutSuffix(name, "...") + if multi && len(rest) != 0 { + return nil, errors.New("{...} wildcard not at end") + } + if name == "" { + return nil, errors.New("empty wildcard") + } + if !http_isValidWildcardName(name) { + return nil, fmt.Errorf("bad wildcard name %q", name) + } + if seenNames[name] { + return nil, fmt.Errorf("duplicate wildcard name %q", name) + } + seenNames[name] = true + p.segments = append(p.segments, http_segment{s: name, wild: true, multi: multi}) + } + } + return p, nil +} + +func http_isValidWildcardName(s string) bool { + if s == "" { return false } - for i, b := range t1 { - if b >= utf8.RuneSelf { - - return false - } - if lowerASCII(byte(b)) != lowerASCII(t2[i]) { + // Valid Go identifier. + for i, c := range s { + if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) { return false } } return true } +func http_pathUnescape(path string) string { + u, err := url.PathUnescape(path) + if err != nil { + // Invalidly escaped path; use the original + return path + } + return u +} + +// relationship is a relationship between two patterns, p1 and p2. +type http_relationship string + const ( - defaultMaxMemory = 32 << 20 // 32 MB + http_equivalent http_relationship = "equivalent" // both match the same requests + http_moreGeneral http_relationship = "moreGeneral" // p1 matches everything p2 does & more + http_moreSpecific http_relationship = "moreSpecific" // p2 matches everything p1 does & more + http_disjoint http_relationship = "disjoint" // there is no request that both match + http_overlaps http_relationship = "overlaps" // there is a request that both match, but neither is more specific +) + +// conflictsWith reports whether p1 conflicts with p2, that is, whether +// there is a request that both match but where neither is higher precedence +// than the other. +// +// Precedence is defined by two rules: +// 1. Patterns with a host win over patterns without a host. +// 2. Patterns whose method and path is more specific win. One pattern is more +// specific than another if the second matches all the (method, path) pairs +// of the first and more. +// +// If rule 1 doesn't apply, then two patterns conflict if their relationship +// is either equivalence (they match the same set of requests) or overlap +// (they both match some requests, but neither is more specific than the other). +func (p1 *http_pattern) conflictsWith(p2 *http_pattern) bool { + if p1.host != p2.host { + // Either one host is empty and the other isn't, in which case the + // one with the host wins by rule 1, or neither host is empty + // and they differ, so they won't match the same paths. + return false + } + rel := p1.comparePathsAndMethods(p2) + return rel == http_equivalent || rel == http_overlaps +} + +func (p1 *http_pattern) comparePathsAndMethods(p2 *http_pattern) http_relationship { + mrel := p1.compareMethods(p2) + // Optimization: avoid a call to comparePaths. + if mrel == http_disjoint { + return http_disjoint + } + prel := p1.comparePaths(p2) + return http_combineRelationships(mrel, prel) +} + +// compareMethods determines the relationship between the method +// part of patterns p1 and p2. +// +// A method can either be empty, "GET", or something else. +// The empty string matches any method, so it is the most general. +// "GET" matches both GET and HEAD. +// Anything else matches only itself. +func (p1 *http_pattern) compareMethods(p2 *http_pattern) http_relationship { + if p1.method == p2.method { + return http_equivalent + } + if p1.method == "" { + // p1 matches any method, but p2 does not, so p1 is more general. + return http_moreGeneral + } + if p2.method == "" { + return http_moreSpecific + } + if p1.method == "GET" && p2.method == "HEAD" { + // p1 matches GET and HEAD; p2 matches only HEAD. + return http_moreGeneral + } + if p2.method == "GET" && p1.method == "HEAD" { + return http_moreSpecific + } + return http_disjoint +} + +// comparePaths determines the relationship between the path +// part of two patterns. +func (p1 *http_pattern) comparePaths(p2 *http_pattern) http_relationship { + // Optimization: if a path pattern doesn't end in a multi ("...") wildcard, then it + // can only match paths with the same number of segments. + if len(p1.segments) != len(p2.segments) && !p1.lastSegment().multi && !p2.lastSegment().multi { + return http_disjoint + } + + // Consider corresponding segments in the two path patterns. + var segs1, segs2 []http_segment + rel := http_equivalent + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + rel = http_combineRelationships(rel, http_compareSegments(segs1[0], segs2[0])) + if rel == http_disjoint { + return rel + } + } + // We've reached the end of the corresponding segments of the patterns. + // If they have the same number of segments, then we've already determined + // their relationship. + if len(segs1) == 0 && len(segs2) == 0 { + return rel + } + // Otherwise, the only way they could fail to be disjoint is if the shorter + // pattern ends in a multi. In that case, that multi is more general + // than the remainder of the longer pattern, so combine those two relationships. + if len(segs1) < len(segs2) && p1.lastSegment().multi { + return http_combineRelationships(rel, http_moreGeneral) + } + if len(segs2) < len(segs1) && p2.lastSegment().multi { + return http_combineRelationships(rel, http_moreSpecific) + } + return http_disjoint +} + +// compareSegments determines the relationship between two segments. +func http_compareSegments(s1, s2 http_segment) http_relationship { + if s1.multi && s2.multi { + return http_equivalent + } + if s1.multi { + return http_moreGeneral + } + if s2.multi { + return http_moreSpecific + } + if s1.wild && s2.wild { + return http_equivalent + } + if s1.wild { + if s2.s == "/" { + // A single wildcard doesn't match a trailing slash. + return http_disjoint + } + return http_moreGeneral + } + if s2.wild { + if s1.s == "/" { + return http_disjoint + } + return http_moreSpecific + } + // Both literals. + if s1.s == s2.s { + return http_equivalent + } + return http_disjoint +} + +// combineRelationships determines the overall relationship of two patterns +// given the relationships of a partition of the patterns into two parts. +// +// For example, if p1 is more general than p2 in one way but equivalent +// in the other, then it is more general overall. +// +// Or if p1 is more general in one way and more specific in the other, then +// they overlap. +func http_combineRelationships(r1, r2 http_relationship) http_relationship { + switch r1 { + case http_equivalent: + return r2 + case http_disjoint: + return http_disjoint + case http_overlaps: + if r2 == http_disjoint { + return http_disjoint + } + return http_overlaps + case http_moreGeneral, http_moreSpecific: + switch r2 { + case http_equivalent: + return r1 + case http_inverseRelationship(r1): + return http_overlaps + default: + return r2 + } + default: + panic(fmt.Sprintf("unknown relationship %q", r1)) + } +} + +// If p1 has relationship ` + "`" + `r` + "`" + ` to p2, then +// p2 has inverseRelationship(r) to p1. +func http_inverseRelationship(r http_relationship) http_relationship { + switch r { + case http_moreSpecific: + return http_moreGeneral + case http_moreGeneral: + return http_moreSpecific + default: + return r + } +} + +// describeConflict returns an explanation of why two patterns conflict. +func http_describeConflict(p1, p2 *http_pattern) string { + mrel := p1.compareMethods(p2) + prel := p1.comparePaths(p2) + rel := http_combineRelationships(mrel, prel) + if rel == http_equivalent { + return fmt.Sprintf("%s matches the same requests as %s", p1, p2) + } + if rel != http_overlaps { + panic("describeConflict called with non-conflicting patterns") + } + if prel == http_overlaps { + return fmt.Sprintf(` + "`" + `%[1]s and %[2]s both match some paths, like %[3]q. +But neither is more specific than the other. +%[1]s matches %[4]q, but %[2]s doesn't. +%[2]s matches %[5]q, but %[1]s doesn't.` + "`" + `, + p1, p2, http_commonPath(p1, p2), http_differencePath(p1, p2), http_differencePath(p2, p1)) + } + if mrel == http_moreGeneral && prel == http_moreSpecific { + return fmt.Sprintf("%s matches more methods than %s, but has a more specific path pattern", p1, p2) + } + if mrel == http_moreSpecific && prel == http_moreGeneral { + return fmt.Sprintf("%s matches fewer methods than %s, but has a more general path pattern", p1, p2) + } + return fmt.Sprintf("bug: unexpected way for two patterns %s and %s to conflict: methods %s, paths %s", p1, p2, mrel, prel) +} + +// writeMatchingPath writes to b a path that matches the segments. +func http_writeMatchingPath(b *strings.Builder, segs []http_segment) { + for _, s := range segs { + http_writeSegment(b, s) + } +} + +func http_writeSegment(b *strings.Builder, s http_segment) { + b.WriteByte('/') + if !s.multi && s.s != "/" { + b.WriteString(s.s) + } +} + +// commonPath returns a path that both p1 and p2 match. +// It assumes there is such a path. +func http_commonPath(p1, p2 *http_pattern) string { + var b strings.Builder + var segs1, segs2 []http_segment + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + if s1 := segs1[0]; s1.wild { + http_writeSegment(&b, segs2[0]) + } else { + http_writeSegment(&b, s1) + } + } + if len(segs1) > 0 { + http_writeMatchingPath(&b, segs1) + } else if len(segs2) > 0 { + http_writeMatchingPath(&b, segs2) + } + return b.String() +} + +// differencePath returns a path that p1 matches and p2 doesn't. +// It assumes there is such a path. +func http_differencePath(p1, p2 *http_pattern) string { + var b strings.Builder + + var segs1, segs2 []http_segment + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + s1 := segs1[0] + s2 := segs2[0] + if s1.multi && s2.multi { + // From here the patterns match the same paths, so we must have found a difference earlier. + b.WriteByte('/') + return b.String() + + } + if s1.multi && !s2.multi { + // s1 ends in a "..." wildcard but s2 does not. + // A trailing slash will distinguish them, unless s2 ends in "{$}", + // in which case any segment will do; prefer the wildcard name if + // it has one. + b.WriteByte('/') + if s2.s == "/" { + if s1.s != "" { + b.WriteString(s1.s) + } else { + b.WriteString("x") + } + } + return b.String() + } + if !s1.multi && s2.multi { + http_writeSegment(&b, s1) + } else if s1.wild && s2.wild { + // Both patterns will match whatever we put here; use + // the first wildcard name. + http_writeSegment(&b, s1) + } else if s1.wild && !s2.wild { + // s1 is a wildcard, s2 is a literal. + // Any segment other than s2.s will work. + // Prefer the wildcard name, but if it's the same as the literal, + // tweak the literal. + if s1.s != s2.s { + http_writeSegment(&b, s1) + } else { + b.WriteByte('/') + b.WriteString(s2.s + "x") + } + } else if !s1.wild && s2.wild { + http_writeSegment(&b, s1) + } else { + // Both are literals. A precondition of this function is that the + // patterns overlap, so they must be the same literal. Use it. + if s1.s != s2.s { + panic(fmt.Sprintf("literals differ: %q and %q", s1.s, s2.s)) + } + http_writeSegment(&b, s1) + } + } + if len(segs1) > 0 { + // p1 is longer than p2, and p2 does not end in a multi. + // Anything that matches the rest of p1 will do. + http_writeMatchingPath(&b, segs1) + } else if len(segs2) > 0 { + http_writeMatchingPath(&b, segs2) + } + return b.String() +} + +const ( + http_defaultMaxMemory = 32 << 20 // 32 MB ) // ErrMissingFile is returned by FormFile when the provided file field name // is either not present in the request or not a file field. -var ErrMissingFile = errors.New("http: no such file") +var http_ErrMissingFile = errors.New("http: no such file") -// HTTP request parsing errors. -type ProtocolError struct { +// ProtocolError represents an HTTP protocol error. +// +// Deprecated: Not all errors in the http package related to protocol errors +// are of type ProtocolError. +type http_ProtocolError struct { ErrorString string } -func (err *ProtocolError) Error() string { return err.ErrorString } +func (pe *http_ProtocolError) Error() string { return pe.ErrorString } -var ( - ErrHeaderTooLong = &ProtocolError{"header too long"} - ErrShortBody = &ProtocolError{"entity body too short"} - ErrNotSupported = &ProtocolError{"feature not supported"} - ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} - ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} - ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} - ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} -) - -type badStringError struct { - what string - str string +// Is lets http.ErrNotSupported match errors.ErrUnsupported. +func (pe *http_ProtocolError) Is(err error) bool { + return pe == http_ErrNotSupported && err == errors.ErrUnsupported } -func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } +var ( + // ErrNotSupported indicates that a feature is not supported. + // + // It is returned by ResponseController methods to indicate that + // the handler does not support the method, and by the Push method + // of Pusher implementations to indicate that HTTP/2 Push support + // is not available. + http_ErrNotSupported = &http_ProtocolError{"feature not supported"} + + // Deprecated: ErrUnexpectedTrailer is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + http_ErrUnexpectedTrailer = &http_ProtocolError{"trailer header without chunked transfer encoding"} + + // ErrMissingBoundary is returned by Request.MultipartReader when the + // request's Content-Type does not include a "boundary" parameter. + http_ErrMissingBoundary = &http_ProtocolError{"no multipart boundary param in Content-Type"} + + // ErrNotMultipart is returned by Request.MultipartReader when the + // request's Content-Type is not multipart/form-data. + http_ErrNotMultipart = &http_ProtocolError{"request Content-Type isn't multipart/form-data"} + + // Deprecated: ErrHeaderTooLong is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + http_ErrHeaderTooLong = &http_ProtocolError{"header too long"} + + // Deprecated: ErrShortBody is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + http_ErrShortBody = &http_ProtocolError{"entity body too short"} + + // Deprecated: ErrMissingContentLength is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + http_ErrMissingContentLength = &http_ProtocolError{"missing ContentLength in HEAD response"} +) + +func http_badStringError(what, val string) error { return fmt.Errorf("%s %q", what, val) } // Headers that Request.Write handles itself and should be skipped. -var reqWriteExcludeHeader = map[string]bool{ - "Host": true, +var http_reqWriteExcludeHeader = map[string]bool{ + "Host": true, // not in Header map anyway "User-Agent": true, "Content-Length": true, "Transfer-Encoding": true, @@ -2026,19 +5565,19 @@ // // The field semantics differ slightly between client and server // usage. In addition to the notes on the fields below, see the -// documentation for Request.Write and RoundTripper. -type Request struct { +// documentation for [Request.Write] and [RoundTripper]. +type http_Request struct { // Method specifies the HTTP method (GET, POST, PUT, etc.). - // For client requests an empty string means GET. + // For client requests, an empty string means GET. Method string // URL specifies either the URI being requested (for server // requests) or the URL to access (for client requests). // - // For server requests the URL is parsed from the URI + // For server requests, the URL is parsed from the URI // supplied on the Request-Line as stored in RequestURI. For // most requests, fields other than Path and RawQuery will be - // empty. (See RFC 2616, Section 5.1.2) + // empty. (See RFC 7230, Section 5.3) // // For client requests, the URL's Host specifies the server to // connect to, while the Request's Host field optionally @@ -2046,55 +5585,79 @@ // request. URL *url.URL - // The protocol version for incoming requests. - // Client requests always use HTTP/1.1. + // The protocol version for incoming server requests. + // + // For client requests, these fields are ignored. The HTTP + // client code always uses either HTTP/1.1 or HTTP/2. + // See the docs on Transport for details. Proto string // "HTTP/1.0" ProtoMajor int // 1 ProtoMinor int // 0 - // A header maps request lines to their values. - // If the header says + // Header contains the request header fields either received + // by the server or to be sent by the client. // + // If a server received a request with header lines, + // + // Host: example.com // accept-encoding: gzip, deflate // Accept-Language: en-us - // Connection: keep-alive + // fOO: Bar + // foo: two // // then // // Header = map[string][]string{ // "Accept-Encoding": {"gzip, deflate"}, // "Accept-Language": {"en-us"}, - // "Connection": {"keep-alive"}, + // "Foo": {"Bar", "two"}, // } // - // HTTP defines that header names are case-insensitive. - // The request parser implements this by canonicalizing the - // name, making the first character and any characters - // following a hyphen uppercase and the rest lowercase. + // For incoming requests, the Host header is promoted to the + // Request.Host field and removed from the Header map. // - // For client requests certain headers are automatically - // added and may override values in Header. + // HTTP defines that header names are case-insensitive. The + // request parser implements this by using CanonicalHeaderKey, + // making the first character and any characters following a + // hyphen uppercase and the rest lowercase. // - // See the documentation for the Request.Write method. - Header Header + // For client requests, certain headers such as Content-Length + // and Connection are automatically written when needed and + // values in Header may be ignored. See the documentation + // for the Request.Write method. + Header http_Header // Body is the request's body. // - // For client requests a nil body means the request has no + // For client requests, a nil body means the request has no // body, such as a GET request. The HTTP Client's Transport // is responsible for calling the Close method. // - // For server requests the Request Body is always non-nil + // For server requests, the Request Body is always non-nil // but will return EOF immediately when no body is present. // The Server will close the request body. The ServeHTTP // Handler does not need to. + // + // Body must allow Read to be called concurrently with Close. + // In particular, calling Close should unblock a Read waiting + // for input. Body io.ReadCloser + // GetBody defines an optional func to return a new copy of + // Body. It is used for client requests when a redirect requires + // reading the body more than once. Use of GetBody still + // requires setting Body. + // + // For server requests, it is unused. + GetBody func() (io.ReadCloser, error) + // ContentLength records the length of the associated content. // The value -1 indicates that the length is unknown. // Values >= 0 indicate that the given number of bytes may // be read from Body. - // For client requests, a value of 0 means unknown if Body is not nil. + // + // For client requests, a value of 0 with a non-nil Body is + // also treated as unknown. ContentLength int64 // TransferEncoding lists the transfer encodings from outermost to @@ -2105,27 +5668,45 @@ TransferEncoding []string // Close indicates whether to close the connection after - // replying to this request (for servers) or after sending - // the request (for clients). + // replying to this request (for servers) or after sending this + // request and reading its response (for clients). + // + // For server requests, the HTTP server handles this automatically + // and this field is not needed by Handlers. + // + // For client requests, setting this field prevents re-use of + // TCP connections between requests to the same hosts, as if + // Transport.DisableKeepAlives were set. Close bool - // For server requests Host specifies the host on which the - // URL is sought. Per RFC 2616, this is either the value of - // the "Host" header or the host name given in the URL itself. - // It may be of the form "host:port". + // For server requests, Host specifies the host on which the + // URL is sought. For HTTP/1 (per RFC 7230, section 5.4), this + // is either the value of the "Host" header or the host name + // given in the URL itself. For HTTP/2, it is the value of the + // ":authority" pseudo-header field. + // It may be of the form "host:port". For international domain + // names, Host may be in Punycode or Unicode form. Use + // golang.org/x/net/idna to convert it to either format if + // needed. + // To prevent DNS rebinding attacks, server Handlers should + // validate that the Host header has a value for which the + // Handler considers itself authoritative. The included + // ServeMux supports patterns registered to particular host + // names and thus protects its registered Handlers. // - // For client requests Host optionally overrides the Host + // For client requests, Host optionally overrides the Host // header to send. If empty, the Request.Write method uses - // the value of URL.Host. + // the value of URL.Host. Host may contain an international + // domain name. Host string // Form contains the parsed form data, including both the URL - // field's query parameters and the POST or PUT form data. + // field's query parameters and the PATCH, POST, or PUT form data. // This field is only available after ParseForm is called. // The HTTP client ignores Form and uses Body instead. Form url.Values - // PostForm contains the parsed form data from POST, PATCH, + // PostForm contains the parsed form data from PATCH, POST // or PUT body parameters. // // This field is only available after ParseForm is called. @@ -2140,22 +5721,26 @@ // Trailer specifies additional headers that are sent after the request // body. // - // For server requests the Trailer map initially contains only the + // For server requests, the Trailer map initially contains only the // trailer keys, with nil values. (The client declares which trailers it // will later send.) While the handler is reading from Body, it must // not reference Trailer. After reading from Body returns EOF, Trailer // can be read again and will contain non-nil values, if they were sent // by the client. // - // For client requests Trailer must be initialized to a map containing + // For client requests, Trailer must be initialized to a map containing // the trailer keys to later send. The values may be nil or their final // values. The ContentLength must be 0 or -1, to send a chunked request. // After the HTTP request is sent the map values can be updated while // the request body is read. Once the body returns EOF, the caller must // not mutate Trailer. // + // Writing a request whose Trailer contains a key with invalid bytes + // (such as CR or LF), or such a value present when Write begins, + // returns an error. + // // Few HTTP clients, servers, or proxies support HTTP trailers. - Trailer Header + Trailer http_Header // RemoteAddr allows HTTP servers and other software to record // the network address that sent the request, usually for @@ -2166,8 +5751,8 @@ // This field is ignored by the HTTP client. RemoteAddr string - // RequestURI is the unmodified Request-URI of the - // Request-Line (RFC 2616, Section 5.1) as sent by the client + // RequestURI is the unmodified request-target of the + // Request-Line (RFC 7230, Section 3.1.1) as sent by the client // to a server. Usually the URL field should be used instead. // It is an error to set this field in an HTTP client request. RequestURI string @@ -2186,44 +5771,158 @@ // RoundTripper may support Cancel. // // For server requests, this field is not applicable. + // + // Deprecated: Set the Request's context with NewRequestWithContext + // instead. If a Request's Cancel field and context are both + // set, it is undefined whether Cancel is respected. Cancel <-chan struct{} + + // Response is the redirect response which caused this request + // to be created. This field is only populated during client + // redirects. + Response *http_Response + + // Pattern is the [ServeMux] pattern that matched the request. + // It is empty if the request was not matched against a pattern. + Pattern string + + // ctx is either the client or server context. It should only + // be modified via copying the whole Request using Clone or WithContext. + // It is unexported to prevent people from using Context wrong + // and mutating the contexts held by callers of the same request. + ctx context.Context + + // The following fields are for requests matched by ServeMux. + pat *http_pattern // the pattern that matched + matches []string // values for the matching wildcards in pat + otherValues map[string]string // for calls to SetPathValue that don't match a wildcard +} + +// Context returns the request's context. To change the context, use +// [Request.Clone] or [Request.WithContext]. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For outgoing client requests, the context controls cancellation. +// +// For incoming server requests, the context is canceled when the +// client's connection closes, the request is canceled (with HTTP/2), +// or when the ServeHTTP method returns. +func (r *http_Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +// +// For outgoing client request, the context controls the entire +// lifetime of a request and its response: obtaining a connection, +// sending the request, and reading the response headers and body. +// +// To create a new request with a context, use [NewRequestWithContext]. +// To make a deep copy of a request with a new context, use [Request.Clone]. +func (r *http_Request) WithContext(ctx context.Context) *http_Request { + if ctx == nil { + panic("nil context") + } + r2 := new(http_Request) + *r2 = *r + r2.ctx = ctx + return r2 +} + +// Clone returns a deep copy of r with its context changed to ctx. +// The provided ctx must be non-nil. +// +// Clone only makes a shallow copy of the Body field. +// +// For an outgoing client request, the context controls the entire +// lifetime of a request and its response: obtaining a connection, +// sending the request, and reading the response headers and body. +func (r *http_Request) Clone(ctx context.Context) *http_Request { + if ctx == nil { + panic("nil context") + } + r2 := new(http_Request) + *r2 = *r + r2.ctx = ctx + r2.URL = http_cloneURL(r.URL) + r2.Header = r.Header.Clone() + r2.Trailer = r.Trailer.Clone() + if s := r.TransferEncoding; s != nil { + s2 := make([]string, len(s)) + copy(s2, s) + r2.TransferEncoding = s2 + } + r2.Form = http_cloneURLValues(r.Form) + r2.PostForm = http_cloneURLValues(r.PostForm) + r2.MultipartForm = http_cloneMultipartForm(r.MultipartForm) + + // Copy matches and otherValues. See issue 61410. + if s := r.matches; s != nil { + s2 := make([]string, len(s)) + copy(s2, s) + r2.matches = s2 + } + r2.otherValues = maps.Clone(r.otherValues) + return r2 } // ProtoAtLeast reports whether the HTTP protocol used // in the request is at least major.minor. -func (r *Request) ProtoAtLeast(major, minor int) bool { +func (r *http_Request) ProtoAtLeast(major, minor int) bool { return r.ProtoMajor > major || r.ProtoMajor == major && r.ProtoMinor >= minor } // UserAgent returns the client's User-Agent, if sent in the request. -func (r *Request) UserAgent() string { +func (r *http_Request) UserAgent() string { return r.Header.Get("User-Agent") } // Cookies parses and returns the HTTP cookies sent with the request. -func (r *Request) Cookies() []*Cookie { - return readCookies(r.Header, "") +func (r *http_Request) Cookies() []*http_Cookie { + return http_readCookies(r.Header, "") +} + +// CookiesNamed parses and returns the named HTTP cookies sent with the request +// or an empty slice if none matched. +func (r *http_Request) CookiesNamed(name string) []*http_Cookie { + if name == "" { + return []*http_Cookie{} + } + return http_readCookies(r.Header, name) } // ErrNoCookie is returned by Request's Cookie method when a cookie is not found. -var ErrNoCookie = errors.New("http: named cookie not present") +var http_ErrNoCookie = errors.New("http: named cookie not present") // Cookie returns the named cookie provided in the request or -// ErrNoCookie if not found. -func (r *Request) Cookie(name string) (*Cookie, error) { - for _, c := range readCookies(r.Header, name) { +// [ErrNoCookie] if not found. +// If multiple cookies match the given name, only one cookie will +// be returned. +func (r *http_Request) Cookie(name string) (*http_Cookie, error) { + if name == "" { + return nil, http_ErrNoCookie + } + for _, c := range http_readCookies(r.Header, name) { return c, nil } - return nil, ErrNoCookie + return nil, http_ErrNoCookie } -// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, -// AddCookie does not attach more than one Cookie header field. That +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one [Cookie] header field. That // means all cookies, if any, are written into the same line, // separated by semicolon. -func (r *Request) AddCookie(c *Cookie) { - s := fmt.Sprintf("%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) +// AddCookie only sanitizes c's name and value, and does not sanitize +// a Cookie header already present in the request. +func (r *http_Request) AddCookie(c *http_Cookie) { + s := fmt.Sprintf("%s=%s", http_sanitizeCookieName(c.Name), http_sanitizeCookieValue(c.Value, c.Quoted)) if c := r.Header.Get("Cookie"); c != "" { r.Header.Set("Cookie", c+"; "+s) } else { @@ -2235,55 +5934,64 @@ // // Referer is misspelled as in the request itself, a mistake from the // earliest days of HTTP. This value can also be fetched from the -// Header map as Header["Referer"]; the benefit of making it available +// [Header] map as Header["Referer"]; the benefit of making it available // as a method is that the compiler can diagnose programs that use the // alternate (correct English) spelling req.Referrer() but cannot // diagnose programs that use Header["Referrer"]. -func (r *Request) Referer() string { +func (r *http_Request) Referer() string { return r.Header.Get("Referer") } // multipartByReader is a sentinel value. // Its presence in Request.MultipartForm indicates that parsing of the request -// body has been handed off to a MultipartReader instead of ParseMultipartFrom. -var multipartByReader = &multipart.Form{ +// body has been handed off to a MultipartReader instead of ParseMultipartForm. +var http_multipartByReader = &multipart.Form{ Value: make(map[string][]string), File: make(map[string][]*multipart.FileHeader), } // MultipartReader returns a MIME multipart reader if this is a -// multipart/form-data POST request, else returns nil and an error. -// Use this function instead of ParseMultipartForm to +// multipart/form-data or a multipart/mixed POST request, else returns nil and an error. +// Use this function instead of [Request.ParseMultipartForm] to // process the request body as a stream. -func (r *Request) MultipartReader() (*multipart.Reader, error) { - if r.MultipartForm == multipartByReader { +func (r *http_Request) MultipartReader() (*multipart.Reader, error) { + if r.MultipartForm == http_multipartByReader { return nil, errors.New("http: MultipartReader called twice") } if r.MultipartForm != nil { return nil, errors.New("http: multipart handled by ParseMultipartForm") } - r.MultipartForm = multipartByReader - return r.multipartReader() + r.MultipartForm = http_multipartByReader + return r.multipartReader(true) } -func (r *Request) multipartReader() (*multipart.Reader, error) { +func (r *http_Request) multipartReader(allowMixed bool) (*multipart.Reader, error) { v := r.Header.Get("Content-Type") if v == "" { - return nil, ErrNotMultipart + return nil, http_ErrNotMultipart + } + if r.Body == nil { + return nil, errors.New("missing form body") } d, params, err := mime.ParseMediaType(v) - if err != nil || d != "multipart/form-data" { - return nil, ErrNotMultipart + if err != nil || !(d == "multipart/form-data" || allowMixed && d == "multipart/mixed") { + return nil, http_ErrNotMultipart } boundary, ok := params["boundary"] if !ok { - return nil, ErrMissingBoundary + return nil, http_ErrMissingBoundary } return multipart.NewReader(r.Body, boundary), nil } +// isH2Upgrade reports whether r represents the http2 "client preface" +// magic string. +func (r *http_Request) isH2Upgrade() bool { + return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" +} + // Return value if nonempty, def otherwise. -func valueOrDefault(value, def string) string { +func http_valueOrDefault(value, def string) string { if value != "" { return value } @@ -2292,12 +6000,13 @@ // NOTE: This is not intended to reflect the actual Go version being used. // It was changed at the time of Go 1.1 release because the former User-Agent -// had ended up on a blacklist for some intrusion detection systems. +// had ended up blocked by some intrusion detection systems. // See https://codereview.appspot.com/7532043. -const defaultUserAgent = "Go-http-client/1.1" +const http_defaultUserAgent = "Go-http-client/1.1" // Write writes an HTTP/1.1 request, which is the header and body, in wire format. // This method consults the following fields of the request: +// // Host // URL // Method (defaults to "GET") @@ -2306,43 +6015,115 @@ // TransferEncoding // Body // -// If Body is present, Content-Length is <= 0 and TransferEncoding +// If Body is present, Content-Length is <= 0 and [Request.TransferEncoding] // hasn't been set to "identity", Write adds "Transfer-Encoding: // chunked" to the header. Body is closed after it is sent. -func (r *Request) Write(w io.Writer) error { - return r.write(w, false, nil) +// +// Header values for Host, Content-Length, Transfer-Encoding, +// and Trailer are not used; these are derived from other Request fields. +// If the Header does not contain a User-Agent value, Write uses +// "Go-http-client/1.1". +func (r *http_Request) Write(w io.Writer) error { + return r.write(w, false, nil, nil) } -// WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. In particular, WriteProxy writes the +// WriteProxy is like [Request.Write] but writes the request in the form +// expected by an HTTP proxy. In particular, [Request.WriteProxy] writes the // initial Request-URI line of the request with an absolute URI, per -// section 5.1.2 of RFC 2616, including the scheme and host. +// section 5.3 of RFC 7230, including the scheme and host. // In either case, WriteProxy also writes a Host header, using // either r.Host or r.URL.Host. -func (r *Request) WriteProxy(w io.Writer) error { - return r.write(w, true, nil) +func (r *http_Request) WriteProxy(w io.Writer) error { + return r.write(w, true, nil, nil) } +// errMissingHost is returned by Write when there is no Host or URL present in +// the Request. +var http_errMissingHost = errors.New("http: Request.Write on Request with no Host or URL set") + // extraHeaders may be nil -func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error { - - host := cleanHost(req.Host) - if host == "" { - if req.URL == nil { - return errors.New("http: Request.Write on Request with no Host or URL set") +// waitForContinue may be nil +// always closes body +func (r *http_Request) write(w io.Writer, usingProxy bool, extraHeaders http_Header, waitForContinue func() bool) (err error) { + trace := httptrace.ContextClientTrace(r.Context()) + if trace != nil && trace.WroteRequest != nil { + defer func() { + trace.WroteRequest(httptrace.WroteRequestInfo{ + Err: err, + }) + }() + } + closed := false + defer func() { + if closed { + return } - host = cleanHost(req.URL.Host) + if closeErr := r.closeBody(); closeErr != nil && err == nil { + err = closeErr + } + }() + + // Find the target host. Prefer the Host: header, but if that + // is not given, use the host from the request URL. + // + // Clean the host, in case it arrives with unexpected stuff in it. + host := r.Host + if host == "" { + if r.URL == nil { + return http_errMissingHost + } + host = r.URL.Host + } + host, err = httpguts.PunycodeHostPort(host) + if err != nil { + return err + } + // Validate that the Host header is a valid header in general, + // but don't validate the host itself. This is sufficient to avoid + // header or request smuggling via the Host field. + // The server can (and will, if it's a net/http server) reject + // the request if it doesn't consider the host valid. + if !httpguts.ValidHostHeader(host) { + // Historically, we would truncate the Host header after '/' or ' '. + // Some users have relied on this truncation to convert a network + // address such as Unix domain socket path into a valid, ignored + // Host header (see https://go.dev/issue/61431). + // + // We don't preserve the truncation, because sending an altered + // header field opens a smuggling vector. Instead, zero out the + // Host header entirely if it isn't valid. (An empty Host is valid; + // see RFC 9112 Section 3.2.) + // + // Return an error if we're sending to a proxy, since the proxy + // probably can't do anything useful with an empty Host header. + if !usingProxy { + host = "" + } else { + return errors.New("http: invalid Host header") + } } - host = removeZone(host) + // According to RFC 6874, an HTTP client, proxy, or other + // intermediary must remove any IPv6 zone identifier attached + // to an outgoing URI. + host = http_removeZone(host) - ruri := req.URL.RequestURI() - if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" { - ruri = req.URL.Scheme + "://" + host + ruri - } else if req.Method == "CONNECT" && req.URL.Path == "" { - + ruri := r.URL.RequestURI() + if usingProxy && r.URL.Scheme != "" && r.URL.Opaque == "" { + ruri = r.URL.Scheme + "://" + host + ruri + } else if r.Method == "CONNECT" && r.URL.Path == "" { + // CONNECT requests normally give just the host and port, not a full URL. ruri = host + if r.URL.Opaque != "" { + ruri = r.URL.Opaque + } } + if http_stringContainsCTLByte(ruri) { + return errors.New("net/http: can't write control character in Request.URL") + } + // TODO: validate r.Method too? At least it's less likely to + // come from an attacker (more likely to be a constant in + // code). // Wrap the writer in a bufio Writer if it's not already buffered. // Don't always call NewWriter, as that forces a bytes.Buffer @@ -2354,45 +6135,55 @@ w = bw } - _, err := fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", http_valueOrDefault(r.Method, "GET"), ruri) if err != nil { return err } + // Header lines _, err = fmt.Fprintf(w, "Host: %s\r\n", host) if err != nil { return err } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Host", []string{host}) + } - userAgent := defaultUserAgent - if req.Header != nil { - if ua := req.Header["User-Agent"]; len(ua) > 0 { - userAgent = ua[0] - } + // Use the defaultUserAgent unless the Header contains one, which + // may be blank to not send the header. + userAgent := http_defaultUserAgent + if r.Header.has("User-Agent") { + userAgent = r.Header.Get("User-Agent") } if userAgent != "" { + userAgent = http_headerNewlineToSpace.Replace(userAgent) + userAgent = textproto.TrimString(userAgent) _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) if err != nil { return err } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("User-Agent", []string{userAgent}) + } } - tw, err := newTransferWriter(req) + // Process Body,ContentLength,Close,Trailer + tw, err := http_newTransferWriter(r) if err != nil { return err } - err = tw.WriteHeader(w) + err = tw.writeHeader(w, trace) if err != nil { return err } - err = req.Header.WriteSubset(w, reqWriteExcludeHeader) + err = r.Header.writeSubset(w, http_reqWriteExcludeHeader, trace) if err != nil { return err } if extraHeaders != nil { - err = extraHeaders.Write(w) + err = extraHeaders.write(w, trace) if err != nil { return err } @@ -2403,8 +6194,41 @@ return err } - err = tw.WriteBody(w) + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } + + // Flush and wait for 100-continue if expected. + if waitForContinue != nil { + if bw, ok := w.(*bufio.Writer); ok { + err = bw.Flush() + if err != nil { + return err + } + } + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } + if !waitForContinue() { + closed = true + r.closeBody() + return nil + } + } + + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + + // Write body and trailer + closed = true + err = tw.writeBody(w) if err != nil { + if tw.bodyReadError == err { + err = http_requestBodyReadError{err} + } return err } @@ -2414,25 +6238,30 @@ return nil } -// cleanHost strips anything after '/' or ' '. -// Ideally we'd clean the Host header according to the spec: -// https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") -// https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) -// https://tools.ietf.org/html/rfc3986#section-3.2.2 (definition of host) -// But practically, what we are trying to avoid is the situation in -// issue 11206, where a malformed Host header used in the proxy context -// would create a bad request. So it is enough to just truncate at the -// first offending character. -func cleanHost(in string) string { - if i := strings.IndexAny(in, " /"); i != -1 { - return in[:i] +// requestBodyReadError wraps an error from (*Request).write to indicate +// that the error came from a Read call on the Request.Body. +// This error type should not escape the net/http package to users. +type http_requestBodyReadError struct{ error } + +func http_idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if ascii.Is(v) { + return v, nil } - return in + return idna.Lookup.ToASCII(v) } -// removeZone removes IPv6 zone identifer from host. +// removeZone removes IPv6 zone identifier from host. // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" -func removeZone(host string) string { +func http_removeZone(host string) string { if !strings.HasPrefix(host, "[") { return host } @@ -2447,10 +6276,10 @@ return host[:j] + host[i:] } -// ParseHTTPVersion parses a HTTP version string. -// "HTTP/1.0" returns (1, 0, true). -func ParseHTTPVersion(vers string) (major, minor int, ok bool) { - const Big = 1000000 // arbitrary upper bound +// ParseHTTPVersion parses an HTTP version string according to RFC 7230, section 2.6. +// "HTTP/1.0" returns (1, 0, true). Note that strings without +// a minor version, such as "HTTP/2", are not valid. +func http_ParseHTTPVersion(vers string) (major, minor int, ok bool) { switch vers { case "HTTP/1.1": return 1, 1, true @@ -2460,35 +6289,82 @@ if !strings.HasPrefix(vers, "HTTP/") { return 0, 0, false } - dot := strings.Index(vers, ".") - if dot < 0 { + if len(vers) != len("HTTP/X.Y") { return 0, 0, false } - major, err := strconv.Atoi(vers[5:dot]) - if err != nil || major < 0 || major > Big { + if vers[6] != '.' { return 0, 0, false } - minor, err = strconv.Atoi(vers[dot+1:]) - if err != nil || minor < 0 || minor > Big { + maj, err := strconv.ParseUint(vers[5:6], 10, 0) + if err != nil { return 0, 0, false } - return major, minor, true + min, err := strconv.ParseUint(vers[7:8], 10, 0) + if err != nil { + return 0, 0, false + } + return int(maj), int(min), true } -// NewRequest returns a new Request given a method, URL, and optional body. +func http_validMethod(method string) bool { + /* + Method = "OPTIONS" ; Section 9.2 + | "GET" ; Section 9.3 + | "HEAD" ; Section 9.4 + | "POST" ; Section 9.5 + | "PUT" ; Section 9.6 + | "DELETE" ; Section 9.7 + | "TRACE" ; Section 9.8 + | "CONNECT" ; Section 9.9 + | extension-method + extension-method = token + token = 1*<any CHAR except CTLs or separators> + */ + return http_isToken(method) +} + +// NewRequest wraps [NewRequestWithContext] using [context.Background]. +func http_NewRequest(method, url string, body io.Reader) (*http_Request, error) { + return http_NewRequestWithContext(context.Background(), method, url, body) +} + +// NewRequestWithContext returns a new [Request] given a method, URL, and +// optional body. // -// If the provided body is also an io.Closer, the returned -// Request.Body is set to body and will be closed by the Client -// methods Do, Post, and PostForm, and Transport.RoundTrip. +// If the provided body is also an [io.Closer], the returned +// [Request.Body] is set to body and will be closed (possibly +// asynchronously) by the Client methods Do, Post, and PostForm, +// and [Transport.RoundTrip]. // -// NewRequest returns a Request suitable for use with Client.Do or -// Transport.RoundTrip. -// To create a request for use with testing a Server Handler use either -// ReadRequest or manually update the Request fields. See the Request -// type's documentation for the difference between inbound and outbound -// request fields. -func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { - u, err := url.Parse(urlStr) +// NewRequestWithContext returns a Request suitable for use with +// [Client.Do] or [Transport.RoundTrip]. To create a request for use with +// testing a Server Handler, either use the [net/http/httptest.NewRequest] function, +// use [ReadRequest], or manually update the Request fields. +// For an outgoing client request, the context +// controls the entire lifetime of a request and its response: +// obtaining a connection, sending the request, and reading the +// response headers and body. See the [Request] type's documentation for +// the difference between inbound and outbound request fields. +// +// If body is of type [*bytes.Buffer], [*bytes.Reader], or +// [*strings.Reader], the returned request's ContentLength is set to its +// exact value (instead of -1), GetBody is populated (so 307 and 308 +// redirects can replay the body), and Body is set to [NoBody] if the +// ContentLength is 0. +func http_NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http_Request, error) { + if method == "" { + // We document that "" means "GET" for Request.Method, and people have + // relied on that from NewRequest, so keep that working. + // We still enforce validMethod for non-empty methods. + method = "GET" + } + if !http_validMethod(method) { + return nil, fmt.Errorf("net/http: invalid method %q", method) + } + if ctx == nil { + return nil, errors.New("net/http: nil Context") + } + u, err := urlpkg.Parse(url) if err != nil { return nil, err } @@ -2496,13 +6372,16 @@ if !ok && body != nil { rc = io.NopCloser(body) } - req := &Request{ + // The host's colon:port should be normalized. See Issue 14836. + u.Host = strings.TrimSuffix(u.Host, ":") + req := &http_Request{ + ctx: ctx, Method: method, URL: u, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: make(Header), + Header: make(http_Header), Body: rc, Host: u.Host, } @@ -2510,10 +6389,43 @@ switch v := body.(type) { case *bytes.Buffer: req.ContentLength = int64(v.Len()) + buf := v.Bytes() + req.GetBody = func() (io.ReadCloser, error) { + r := bytes.NewReader(buf) + return io.NopCloser(r), nil + } case *bytes.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return io.NopCloser(&r), nil + } case *strings.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return io.NopCloser(&r), nil + } + default: + // This is where we'd set it to -1 (at least + // if body != NoBody) to mean unknown, but + // that broke people during the Go 1.8 testing + // period. People depend on it being 0 I + // guess. Maybe retry later. See Issue 18117. + } + // For client requests, Request.ContentLength of 0 + // means either actually 0, or unknown. The only way + // to explicitly say that the ContentLength is zero is + // to set the Body to nil. But turns out too much code + // depends on NewRequest returning a non-nil Body, + // so we use a well-known ReadCloser variable instead + // and have the http package also treat that sentinel + // variable to mean explicitly zero. + if req.GetBody != nil && req.ContentLength == 0 { + req.Body = http_NoBody + req.GetBody = func() (io.ReadCloser, error) { return http_NoBody, nil } } } @@ -2523,57 +6435,73 @@ // BasicAuth returns the username and password provided in the request's // Authorization header, if the request uses HTTP Basic Authentication. // See RFC 2617, Section 2. -func (r *Request) BasicAuth() (username, password string, ok bool) { +func (r *http_Request) BasicAuth() (username, password string, ok bool) { auth := r.Header.Get("Authorization") if auth == "" { - return + return "", "", false } - return parseBasicAuth(auth) + return http_parseBasicAuth(auth) } // parseBasicAuth parses an HTTP Basic Authentication string. // "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). -func parseBasicAuth(auth string) (username, password string, ok bool) { +// +// parseBasicAuth should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/sagernet/sing +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname parseBasicAuth +func http_parseBasicAuth(auth string) (username, password string, ok bool) { const prefix = "Basic " - if !strings.HasPrefix(auth, prefix) { - return + // Case insensitive prefix match. See Issue 22736. + if len(auth) < len(prefix) || !ascii.EqualFold(auth[:len(prefix)], prefix) { + return "", "", false } c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) if err != nil { - return + return "", "", false } cs := string(c) - s := strings.IndexByte(cs, ':') - if s < 0 { - return + username, password, ok = strings.Cut(cs, ":") + if !ok { + return "", "", false } - return cs[:s], cs[s+1:], true + return username, password, true } // SetBasicAuth sets the request's Authorization header to use HTTP // Basic Authentication with the provided username and password. // // With HTTP Basic Authentication the provided username and password -// are not encrypted. -func (r *Request) SetBasicAuth(username, password string) { - r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) +// are not encrypted. It should generally only be used in an HTTPS +// request. +// +// The username may not contain a colon. Some protocols may impose +// additional requirements on pre-escaping the username and +// password. For instance, when used with OAuth2, both arguments must +// be URL encoded first with [url.QueryEscape]. +func (r *http_Request) SetBasicAuth(username, password string) { + r.Header.Set("Authorization", "Basic "+http_basicAuth(username, password)) } // parseRequestLine parses "GET /foo HTTP/1.1" into its three parts. -func parseRequestLine(line string) (method, requestURI, proto string, ok bool) { - s1 := strings.Index(line, " ") - s2 := strings.Index(line[s1+1:], " ") - if s1 < 0 || s2 < 0 { - return +func http_parseRequestLine(line string) (method, requestURI, proto string, ok bool) { + method, rest, ok1 := strings.Cut(line, " ") + requestURI, proto, ok2 := strings.Cut(rest, " ") + if !ok1 || !ok2 { + return "", "", "", false } - s2 += s1 + 1 - return line[:s1], line[s1+1 : s2], line[s2+1:], true + return method, requestURI, proto, true } -var textprotoReaderPool sync.Pool +var http_textprotoReaderPool sync.Pool -func newTextprotoReader(br *bufio.Reader) *textproto.Reader { - if v := textprotoReaderPool.Get(); v != nil { +func http_newTextprotoReader(br *bufio.Reader) *textproto.Reader { + if v := http_textprotoReaderPool.Get(); v != nil { tr := v.(*textproto.Reader) tr.R = br return tr @@ -2581,16 +6509,52 @@ return textproto.NewReader(br) } -func putTextprotoReader(r *textproto.Reader) { +func http_putTextprotoReader(r *textproto.Reader) { r.R = nil - textprotoReaderPool.Put(r) + http_textprotoReaderPool.Put(r) } // ReadRequest reads and parses an incoming request from b. -func ReadRequest(b *bufio.Reader) (req *Request, err error) { +// +// ReadRequest is a low-level function and should only be used for +// specialized applications; most code should use the [Server] to read +// requests and handle them via the [Handler] interface. ReadRequest +// only supports HTTP/1.x requests. For HTTP/2, use golang.org/x/net/http2. +func http_ReadRequest(b *bufio.Reader) (*http_Request, error) { + req, err := http_readRequest(b) + if err != nil { + return nil, err + } - tp := newTextprotoReader(b) - req = new(Request) + delete(req.Header, "Host") + return req, nil +} + +// readMIMEHeader is defined in package [net/textproto]. +// +//go:linkname readMIMEHeader net/textproto.readMIMEHeader +func http_readMIMEHeader(r *textproto.Reader, maxMemory, maxHeaders int64) (textproto.MIMEHeader, error) + +// readRequest should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/sagernet/sing +// - github.com/v2fly/v2ray-core/v4 +// - github.com/v2fly/v2ray-core/v5 +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname readRequest +func http_readRequest(b *bufio.Reader) (req *http_Request, err error) { + return http_readRequestLimit(b, math.MaxInt64) +} + +func http_readRequestLimit(b *bufio.Reader, maxHeaders int64) (req *http_Request, err error) { + tp := http_newTextprotoReader(b) + defer http_putTextprotoReader(tp) + + req = new(http_Request) // First line: GET /index.html HTTP/1.0 var s string @@ -2598,22 +6562,33 @@ return nil, err } defer func() { - putTextprotoReader(tp) if err == io.EOF { err = io.ErrUnexpectedEOF } }() var ok bool - req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s) + req.Method, req.RequestURI, req.Proto, ok = http_parseRequestLine(s) if !ok { - return nil, &badStringError{"malformed HTTP request", s} + return nil, http_badStringError("malformed HTTP request", s) + } + if !http_validMethod(req.Method) { + return nil, http_badStringError("invalid method", req.Method) } rawurl := req.RequestURI - if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { - return nil, &badStringError{"malformed HTTP version", req.Proto} + if req.ProtoMajor, req.ProtoMinor, ok = http_ParseHTTPVersion(req.Proto); !ok { + return nil, http_badStringError("malformed HTTP version", req.Proto) } + // CONNECT requests are used two different ways, and neither uses a full URL: + // The standard use is to tunnel HTTPS through an HTTP proxy. + // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is + // just the authority section of a URL. This information should go in req.URL.Host. + // + // The net/rpc package also uses CONNECT, but there the parameter is a path + // that starts with a slash. It can be parsed with the regular URL parser, + // and the path will end up in req.URL.Path, where it needs to be in order for + // RPC to work. justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/") if justAuthority { rawurl = "http://" + rawurl @@ -2624,113 +6599,153 @@ } if justAuthority { - + // Strip the bogus "http://" back off. req.URL.Scheme = "" } - mimeHeader, err := tp.ReadMIMEHeader() + // Subsequent lines: Key: value. + mimeHeader, err := http_readMIMEHeader(tp, math.MaxInt64, maxHeaders) if err != nil { + // TODO: Add a distinguishable error to net/textproto. + if err.Error() == "message too large" { + return nil, http_errTooLarge + } return nil, err } - req.Header = Header(mimeHeader) + req.Header = http_Header(mimeHeader) + if len(req.Header["Host"]) > 1 { + return nil, fmt.Errorf("too many Host headers") + } + // RFC 7230, section 5.3: Must treat + // GET /index.html HTTP/1.1 + // Host: www.google.com + // and + // GET http://www.google.com/index.html HTTP/1.1 + // Host: doesntmatter + // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { req.Host = req.Header.get("Host") } - delete(req.Header, "Host") - fixPragmaCacheControl(req.Header) + http_fixPragmaCacheControl(req.Header) - req.Close = shouldClose(req.ProtoMajor, req.ProtoMinor, req.Header, false) + req.Close = http_shouldClose(req.ProtoMajor, req.ProtoMinor, req.Header, false) - err = readTransfer(req, b) + err = http_readTransfer(req, b, maxHeaders) if err != nil { return nil, err } + if req.isH2Upgrade() { + // Because it's neither chunked, nor declared: + req.ContentLength = -1 + + // We want to give handlers a chance to hijack the + // connection, but we need to prevent the Server from + // dealing with the connection further if it's not + // hijacked. Set Close to ensure that: + req.Close = true + } return req, nil } -// MaxBytesReader is similar to io.LimitReader but is intended for +// MaxBytesReader is similar to [io.LimitReader] but is intended for // limiting the size of incoming request bodies. In contrast to // io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a -// non-EOF error for a Read beyond the limit, and closes the -// underlying reader when its Close method is called. +// non-nil error of type [*MaxBytesError] for a Read beyond the limit, +// and closes the underlying reader when its Close method is called. // // MaxBytesReader prevents clients from accidentally or maliciously -// sending a large request and wasting server resources. -func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { - return &maxBytesReader{w: w, r: r, n: n} -} - -type maxBytesReader struct { - w ResponseWriter - r io.ReadCloser // underlying reader - n int64 // max bytes remaining - stopped bool - sawEOF bool -} - -func (l *maxBytesReader) tooLarge() (n int, err error) { - if !l.stopped { - l.stopped = true - if res, ok := l.w.(*response); ok { - res.requestTooLarge() - } +// sending a large request and wasting server resources. If possible, +// it tells the [ResponseWriter] to close the connection after the limit +// has been reached. +func http_MaxBytesReader(w http_ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { + if n < 0 { // Treat negative limits as equivalent to 0. + n = 0 } - return 0, errors.New("http: request body too large") + return &http_maxBytesReader{w: w, r: r, i: n, n: n} } -func (l *maxBytesReader) Read(p []byte) (n int, err error) { - toRead := l.n - if l.n == 0 { - if l.sawEOF { - return l.tooLarge() - } +// MaxBytesError is returned by [MaxBytesReader] when its read limit is exceeded. +type http_MaxBytesError struct { + Limit int64 +} - toRead = 1 +func (e *http_MaxBytesError) Error() string { + // Due to Hyrum's law, this text cannot be changed. + return "http: request body too large" +} + +type http_maxBytesReader struct { + w http_ResponseWriter + r io.ReadCloser // underlying reader + i int64 // max bytes initially, for MaxBytesError + n int64 // max bytes remaining + err error // sticky error +} + +func (l *http_maxBytesReader) Read(p []byte) (n int, err error) { + if l.err != nil { + return 0, l.err } - if int64(len(p)) > toRead { - p = p[:toRead] + if len(p) == 0 { + return 0, nil + } + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + // 0 < len(p) < 2^63 + if int64(len(p))-1 > l.n { + p = p[:l.n+1] } n, err = l.r.Read(p) - if err == io.EOF { - l.sawEOF = true - } - if l.n == 0 { - if n > 0 { - return l.tooLarge() - } - return 0, err + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err } - l.n -= int64(n) - if l.n < 0 { - l.n = 0 + + n = int(l.n) + l.n = 0 + + // The server code and client code both use + // maxBytesReader. This "requestTooLarge" check is + // only used by the server code. To prevent binaries + // which only using the HTTP Client code (such as + // cmd/go) from also linking in the HTTP server, don't + // use a static type assertion to the server + // "*response" type. Check this interface instead: + type requestTooLarger interface { + requestTooLarge() } - return + if res, ok := l.w.(requestTooLarger); ok { + res.requestTooLarge() + } + l.err = &http_MaxBytesError{l.i} + return n, l.err } -func (l *maxBytesReader) Close() error { +func (l *http_maxBytesReader) Close() error { return l.r.Close() } -func copyValues(dst, src url.Values) { +func http_copyValues(dst, src url.Values) { for k, vs := range src { - for _, value := range vs { - dst.Add(k, value) - } + dst[k] = append(dst[k], vs...) } } -func parsePostForm(r *Request) (vs url.Values, err error) { +func http_parsePostForm(r *http_Request) (vs url.Values, err error) { if r.Body == nil { err = errors.New("missing form body") return } ct := r.Header.Get("Content-Type") - + // RFC 7231, section 3.1.1.5 - empty type + // MAY be treated as application/octet-stream if ct == "" { ct = "application/octet-stream" } @@ -2739,8 +6754,8 @@ case ct == "application/x-www-form-urlencoded": var reader io.Reader = r.Body maxFormSize := int64(1<<63 - 1) - if _, ok := r.Body.(*maxBytesReader); !ok { - maxFormSize = int64(10 << 20) + if _, ok := r.Body.(*http_maxBytesReader); !ok { + maxFormSize = int64(10 << 20) // 10 MB is a lot of text. reader = io.LimitReader(r.Body, maxFormSize+1) } b, e := io.ReadAll(reader) @@ -2759,28 +6774,39 @@ err = e } case ct == "multipart/form-data": - + // handled by ParseMultipartForm (which is calling us, or should be) + // TODO(bradfitz): there are too many possible + // orders to call too many functions here. + // Clean this up and write more tests. + // request_test.go contains the start of this, + // in TestParseMultipartFormOrder and others. } return } -// ParseForm parses the raw query from the URL and updates r.Form. +// ParseForm populates r.Form and r.PostForm. // -// For POST or PUT requests, it also parses the request body as a form and -// put the results into both r.PostForm and r.Form. -// POST and PUT body parameters take precedence over URL query string values -// in r.Form. +// For all requests, ParseForm parses the raw query from the URL and updates +// r.Form. // -// If the request Body's size has not already been limited by MaxBytesReader, +// For POST, PUT, and PATCH requests, it also reads the request body, parses it +// as a form and puts the results into both r.PostForm and r.Form. Request body +// parameters take precedence over URL query string values in r.Form. +// +// If the request Body's size has not already been limited by [MaxBytesReader], // the size is capped at 10MB. // -// ParseMultipartForm calls ParseForm automatically. -// It is idempotent. -func (r *Request) ParseForm() error { +// For other HTTP methods, or when the Content-Type is not +// application/x-www-form-urlencoded, the request Body is not read, and +// r.PostForm is initialized to a non-nil, empty value. +// +// [Request.ParseMultipartForm] calls ParseForm automatically. +// ParseForm is idempotent. +func (r *http_Request) ParseForm() error { var err error if r.PostForm == nil { if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" { - r.PostForm, err = parsePostForm(r) + r.PostForm, err = http_parsePostForm(r) } if r.PostForm == nil { r.PostForm = make(url.Values) @@ -2789,7 +6815,7 @@ if r.Form == nil { if len(r.PostForm) > 0 { r.Form = make(url.Values) - copyValues(r.Form, r.PostForm) + http_copyValues(r.Form, r.PostForm) } var newValues url.Values if r.URL != nil { @@ -2805,7 +6831,7 @@ if r.Form == nil { r.Form = newValues } else { - copyValues(r.Form, newValues) + http_copyValues(r.Form, newValues) } } return err @@ -2815,23 +6841,25 @@ // The whole request body is parsed and up to a total of maxMemory bytes of // its file parts are stored in memory, with the remainder stored on // disk in temporary files. -// ParseMultipartForm calls ParseForm if necessary. +// ParseMultipartForm calls [Request.ParseForm] if necessary. +// If ParseForm returns an error, ParseMultipartForm returns it but also +// continues parsing the request body. // After one call to ParseMultipartForm, subsequent calls have no effect. -func (r *Request) ParseMultipartForm(maxMemory int64) error { - if r.MultipartForm == multipartByReader { +func (r *http_Request) ParseMultipartForm(maxMemory int64) error { + if r.MultipartForm == http_multipartByReader { return errors.New("http: multipart handled by MultipartReader") } + var parseFormErr error if r.Form == nil { - err := r.ParseForm() - if err != nil { - return err - } + // Let errors in ParseForm fall through, and just + // return it at the end. + parseFormErr = r.ParseForm() } if r.MultipartForm != nil { return nil } - mr, err := r.multipartReader() + mr, err := r.multipartReader(false) if err != nil { return err } @@ -2840,24 +6868,35 @@ if err != nil { return err } + + if r.PostForm == nil { + r.PostForm = make(url.Values) + } for k, v := range f.Value { r.Form[k] = append(r.Form[k], v...) + // r.PostForm should also be populated. See Issue 9305. + r.PostForm[k] = append(r.PostForm[k], v...) } + r.MultipartForm = f - return nil + return parseFormErr } // FormValue returns the first value for the named component of the query. -// POST and PUT body parameters take precedence over URL query string values. -// FormValue calls ParseMultipartForm and ParseForm if necessary and ignores -// any errors returned by these functions. +// The precedence order: +// 1. application/x-www-form-urlencoded form body (POST, PUT, PATCH only) +// 2. query parameters (always) +// 3. multipart/form-data form body (always) +// +// FormValue calls [Request.ParseMultipartForm] and [Request.ParseForm] +// if necessary and ignores any errors returned by these functions. // If key is not present, FormValue returns the empty string. // To access multiple values of the same key, call ParseForm and -// then inspect Request.Form directly. -func (r *Request) FormValue(key string) string { +// then inspect [Request.Form] directly. +func (r *http_Request) FormValue(key string) string { if r.Form == nil { - r.ParseMultipartForm(defaultMaxMemory) + r.ParseMultipartForm(http_defaultMaxMemory) } if vs := r.Form[key]; len(vs) > 0 { return vs[0] @@ -2865,14 +6904,14 @@ return "" } -// PostFormValue returns the first value for the named component of the POST -// or PUT request body. URL query parameters are ignored. -// PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// PostFormValue returns the first value for the named component of the POST, +// PUT, or PATCH request body. URL query parameters are ignored. +// PostFormValue calls [Request.ParseMultipartForm] and [Request.ParseForm] if necessary and ignores // any errors returned by these functions. // If key is not present, PostFormValue returns the empty string. -func (r *Request) PostFormValue(key string) string { +func (r *http_Request) PostFormValue(key string) string { if r.PostForm == nil { - r.ParseMultipartForm(defaultMaxMemory) + r.ParseMultipartForm(http_defaultMaxMemory) } if vs := r.PostForm[key]; len(vs) > 0 { return vs[0] @@ -2881,13 +6920,13 @@ } // FormFile returns the first file for the provided form key. -// FormFile calls ParseMultipartForm and ParseForm if necessary. -func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { - if r.MultipartForm == multipartByReader { +// FormFile calls [Request.ParseMultipartForm] and [Request.ParseForm] if necessary. +func (r *http_Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { + if r.MultipartForm == http_multipartByReader { return nil, nil, errors.New("http: multipart handled by MultipartReader") } if r.MultipartForm == nil { - err := r.ParseMultipartForm(defaultMaxMemory) + err := r.ParseMultipartForm(http_defaultMaxMemory) if err != nil { return nil, nil, err } @@ -2898,31 +6937,133 @@ return f, fhs[0], err } } - return nil, nil, ErrMissingFile + return nil, nil, http_ErrMissingFile } -func (r *Request) expectsContinue() bool { - return hasToken(r.Header.get("Expect"), "100-continue") +// PathValue returns the value for the named path wildcard in the [ServeMux] pattern +// that matched the request. +// It returns the empty string if the request was not matched against a pattern +// or there is no such wildcard in the pattern. +// +// The value is unescaped. For example, if the pattern "/b/{bucket}" matches +// the path "/b/a%2fb", PathValue("bucket") returns "a/b". +func (r *http_Request) PathValue(name string) string { + if i := r.patIndex(name); i >= 0 { + return r.matches[i] + } + return r.otherValues[name] } -func (r *Request) wantsHttp10KeepAlive() bool { +// SetPathValue sets name to value, so that subsequent calls to r.PathValue(name) +// return value. +// It does not unescape value. +func (r *http_Request) SetPathValue(name, value string) { + if i := r.patIndex(name); i >= 0 { + r.matches[i] = value + } else { + if r.otherValues == nil { + r.otherValues = map[string]string{} + } + r.otherValues[name] = value + } +} + +// patIndex returns the index of name in the list of named wildcards of the +// request's pattern, or -1 if there is no such name. +func (r *http_Request) patIndex(name string) int { + // The linear search seems expensive compared to a map, but just creating the map + // takes a lot of time, and most patterns will just have a couple of wildcards. + if r.pat == nil { + return -1 + } + i := 0 + for _, seg := range r.pat.segments { + if seg.wild && seg.s != "" { + if name == seg.s { + return i + } + i++ + } + } + return -1 +} + +func (r *http_Request) expectsContinue() bool { + return http_hasToken(r.Header.get("Expect"), "100-continue") +} + +func (r *http_Request) wantsHttp10KeepAlive() bool { if r.ProtoMajor != 1 || r.ProtoMinor != 0 { return false } - return hasToken(r.Header.get("Connection"), "keep-alive") + return http_hasToken(r.Header.get("Connection"), "keep-alive") } -func (r *Request) wantsClose() bool { - return hasToken(r.Header.get("Connection"), "close") -} - -func (r *Request) closeBody() { - if r.Body != nil { - r.Body.Close() +func (r *http_Request) wantsClose() bool { + if r.Close { + return true } + return http_hasToken(r.Header.get("Connection"), "close") } -var respExcludeHeader = map[string]bool{ +func (r *http_Request) closeBody() error { + if r.Body == nil { + return nil + } + return r.Body.Close() +} + +func (r *http_Request) isReplayable() bool { + if r.Body == nil || r.Body == http_NoBody || r.GetBody != nil { + switch http_valueOrDefault(r.Method, "GET") { + case "GET", "HEAD", "OPTIONS", "TRACE": + return true + } + // The Idempotency-Key, while non-standard, is widely used to + // mean a POST or other request is idempotent. See + // https://golang.org/issue/19943#issuecomment-421092421 + if r.Header.has("Idempotency-Key") || r.Header.has("X-Idempotency-Key") { + return true + } + } + return false +} + +// outgoingLength reports the Content-Length of this outgoing (Client) request. +// It maps 0 into -1 (unknown) when the Body is non-nil. +func (r *http_Request) outgoingLength() int64 { + if r.Body == nil || r.Body == http_NoBody { + return 0 + } + if r.ContentLength != 0 { + return r.ContentLength + } + return -1 +} + +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func http_requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true + } + return false +} + +// requiresHTTP1 reports whether this request requires being sent on +// an HTTP/1 connection. +func (r *http_Request) requiresHTTP1() bool { + return http_hasToken(r.Header.Get("Connection"), "upgrade") && + ascii.EqualFold(r.Header.Get("Upgrade"), "websocket") +} + +var http_respExcludeHeader = map[string]bool{ "Content-Length": true, "Transfer-Encoding": true, "Trailer": true, @@ -2930,39 +7071,53 @@ // Response represents the response from an HTTP request. // -type Response struct { +// The [Client] and [Transport] return Responses from servers once +// the response headers have been received. The response body +// is streamed on demand as the Body field is read. +type http_Response struct { Status string // e.g. "200 OK" StatusCode int // e.g. 200 Proto string // e.g. "HTTP/1.0" ProtoMajor int // e.g. 1 ProtoMinor int // e.g. 0 - // Header maps header keys to values. If the response had multiple + // Header maps header keys to values. If the response had multiple // headers with the same key, they may be concatenated, with comma - // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers - // be semantically equivalent to a comma-delimited sequence.) Values - // duplicated by other fields in this struct (e.g., ContentLength) are - // omitted from Header. + // delimiters. (RFC 7230, section 3.2.2 requires that multiple headers + // be semantically equivalent to a comma-delimited sequence.) When + // Header values are duplicated by other fields in this struct (e.g., + // ContentLength, TransferEncoding, Trailer), the field values are + // authoritative. // // Keys in the map are canonicalized (see CanonicalHeaderKey). - Header Header + Header http_Header // Body represents the response body. // + // The response body is streamed on demand as the Body field + // is read. If the network connection fails or the server + // terminates the response, Body.Read calls return an error. + // // The http Client and Transport guarantee that Body is always // non-nil, even on responses without a body or responses with // a zero-length body. It is the caller's responsibility to - // close Body. The default HTTP client's Transport does not - // attempt to reuse HTTP/1.0 or HTTP/1.1 TCP connections - // ("keep-alive") unless the Body is read to completion and is - // closed. + // close Body. The default HTTP client's Transport may not + // reuse HTTP/1.x "keep-alive" TCP connections if the Body is + // not read to completion and closed; however, manually reading + // the body to completion should not be needed in most cases, + // as closing the body will also cause the body to be read to + // completion asynchronously, up to a conservative limit. // // The Body is automatically dechunked if the server replied // with a "chunked" Transfer-Encoding. + // + // As of Go 1.12, the Body will also implement io.Writer + // on a successful "101 Switching Protocols" response, + // as used by WebSockets and HTTP/2's "h2c" mode. Body io.ReadCloser - // ContentLength records the length of the associated content. The - // value -1 indicates that the length is unknown. Unless Request.Method + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless Request.Method // is "HEAD", values >= 0 indicate that the given number of bytes may // be read from Body. ContentLength int64 @@ -2972,18 +7127,37 @@ TransferEncoding []string // Close records whether the header directed that the connection be - // closed after reading Body. The value is advice for clients: neither + // closed after reading Body. The value is advice for clients: neither // ReadResponse nor Response.Write ever closes a connection. Close bool - // Trailer maps trailer keys to values, in the same - // format as the header. - Trailer Header + // Uncompressed reports whether the response was sent compressed but + // was decompressed by the http package. When true, reading from + // Body yields the uncompressed content instead of the compressed + // content actually set from the server, ContentLength is set to -1, + // and the "Content-Length" and "Content-Encoding" fields are deleted + // from the responseHeader. To get the original response from + // the server, set Transport.DisableCompression to true. + Uncompressed bool - // The Request that was sent to obtain this Response. + // Trailer maps trailer keys to values in the same + // format as Header. + // + // The Trailer initially contains only nil values, one for + // each key specified in the server's "Trailer" header + // value. Those values are not added to Header. + // + // Trailer must not be accessed concurrently with Read calls + // on the Body. + // + // After Body.Read has returned io.EOF, Trailer will contain + // any trailer values sent by the server. + Trailer http_Header + + // Request is the request that was sent to obtain this Response. // Request's Body is nil (having already been consumed). // This is only populated for Client requests. - Request *Request + Request *http_Request // TLS contains information about the TLS connection on which the // response was received. It is nil for unencrypted responses. @@ -2993,22 +7167,22 @@ } // Cookies parses and returns the cookies set in the Set-Cookie headers. -func (r *Response) Cookies() []*Cookie { - return readSetCookies(r.Header) +func (r *http_Response) Cookies() []*http_Cookie { + return http_readSetCookies(r.Header) } -// ErrNoLocation is returned by Response's Location method +// ErrNoLocation is returned by the [Response.Location] method // when no Location header is present. -var ErrNoLocation = errors.New("http: no Location header in response") +var http_ErrNoLocation = errors.New("http: no Location header in response") // Location returns the URL of the response's "Location" header, -// if present. Relative redirects are resolved relative to -// the Response's Request. ErrNoLocation is returned if no +// if present. Relative redirects are resolved relative to +// [Response.Request]. [ErrNoLocation] is returned if no // Location header is present. -func (r *Response) Location() (*url.URL, error) { +func (r *http_Response) Location() (*url.URL, error) { lv := r.Header.Get("Location") if lv == "" { - return nil, ErrNoLocation + return nil, http_ErrNoLocation } if r.Request != nil && r.Request.URL != nil { return r.Request.URL.Parse(lv) @@ -3017,17 +7191,18 @@ } // ReadResponse reads and returns an HTTP response from r. -// The req parameter optionally specifies the Request that corresponds -// to this Response. If nil, a GET request is assumed. +// The req parameter optionally specifies the [Request] that corresponds +// to this [Response]. If nil, a GET request is assumed. // Clients must call resp.Body.Close when finished reading resp.Body. // After that call, clients can inspect resp.Trailer to find key/value // pairs included in the response trailer. -func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { +func http_ReadResponse(r *bufio.Reader, req *http_Request) (*http_Response, error) { tp := textproto.NewReader(r) - resp := &Response{ + resp := &http_Response{ Request: req, } + // Parse the first line of the response. line, err := tp.ReadLine() if err != nil { if err == io.EOF { @@ -3035,26 +7210,26 @@ } return nil, err } - f := strings.SplitN(line, " ", 3) - if len(f) < 2 { - return nil, &badStringError{"malformed HTTP response", line} + proto, status, ok := strings.Cut(line, " ") + if !ok { + return nil, http_badStringError("malformed HTTP response", line) } - reasonPhrase := "" - if len(f) > 2 { - reasonPhrase = f[2] + resp.Proto = proto + resp.Status = strings.TrimLeft(status, " ") + + statusCode, _, _ := strings.Cut(resp.Status, " ") + if len(statusCode) != 3 { + return nil, http_badStringError("malformed HTTP status code", statusCode) } - resp.Status = f[1] + " " + reasonPhrase - resp.StatusCode, err = strconv.Atoi(f[1]) - if err != nil { - return nil, &badStringError{"malformed HTTP status code", f[1]} + resp.StatusCode, err = strconv.Atoi(statusCode) + if err != nil || resp.StatusCode < 0 { + return nil, http_badStringError("malformed HTTP status code", statusCode) + } + if resp.ProtoMajor, resp.ProtoMinor, ok = http_ParseHTTPVersion(resp.Proto); !ok { + return nil, http_badStringError("malformed HTTP version", resp.Proto) } - resp.Proto = f[0] - var ok bool - if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { - return nil, &badStringError{"malformed HTTP version", resp.Proto} - } - + // Parse the response headers. mimeHeader, err := tp.ReadMIMEHeader() if err != nil { if err == io.EOF { @@ -3062,11 +7237,11 @@ } return nil, err } - resp.Header = Header(mimeHeader) + resp.Header = http_Header(mimeHeader) - fixPragmaCacheControl(resp.Header) + http_fixPragmaCacheControl(resp.Header) - err = readTransfer(resp, r) + err = http_readTransfer(resp, r, math.MaxInt64) if err != nil { return nil, err } @@ -3074,11 +7249,14 @@ return resp, nil } -// RFC2616: Should treat +// RFC 7234, section 5.4: Should treat +// // Pragma: no-cache +// // like +// // Cache-Control: no-cache -func fixPragmaCacheControl(header Header) { +func http_fixPragmaCacheControl(header http_Header) { if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { if _, presentcc := header["Cache-Control"]; !presentcc { header["Cache-Control"] = []string{"no-cache"} @@ -3088,45 +7266,47 @@ // ProtoAtLeast reports whether the HTTP protocol used // in the response is at least major.minor. -func (r *Response) ProtoAtLeast(major, minor int) bool { +func (r *http_Response) ProtoAtLeast(major, minor int) bool { return r.ProtoMajor > major || r.ProtoMajor == major && r.ProtoMinor >= minor } -// Write writes r to w in the HTTP/1.n server response format, +// Write writes r to w in the HTTP/1.x server response format, // including the status line, headers, body, and optional trailer. // // This method consults the following fields of the response r: // -// StatusCode -// ProtoMajor -// ProtoMinor -// Request.Method -// TransferEncoding -// Trailer -// Body -// ContentLength -// Header, values for non-canonical keys will have unpredictable behavior +// StatusCode +// ProtoMajor +// ProtoMinor +// Request.Method +// TransferEncoding +// Trailer +// Body +// ContentLength +// Header, values for non-canonical keys will have unpredictable behavior // // The Response Body is closed after it is sent. -func (r *Response) Write(w io.Writer) error { - +func (r *http_Response) Write(w io.Writer) error { + // Status line text := r.Status if text == "" { - var ok bool - text, ok = statusText[r.StatusCode] - if !ok { + text = http_StatusText(r.StatusCode) + if text == "" { text = "status code " + strconv.Itoa(r.StatusCode) } + } else { + // Just to reduce stutter, if user set r.Status to "200 OK" and StatusCode to 200. + // Not important. + text = strings.TrimPrefix(text, strconv.Itoa(r.StatusCode)+" ") } - protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor) - statusCode := strconv.Itoa(r.StatusCode) + " " - text = strings.TrimPrefix(text, statusCode) - if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil { + + if _, err := fmt.Fprintf(w, "HTTP/%d.%d %03d %s\r\n", r.ProtoMajor, r.ProtoMinor, r.StatusCode, text); err != nil { return err } - r1 := new(Response) + // Clone it, so we can modify r1 as needed. + r1 := new(http_Response) *r1 = *r if r1.ContentLength == 0 && r1.Body != nil { // Is it actually 0 length? Or just unknown? @@ -3136,8 +7316,9 @@ return err } if n == 0 { - - r1.Body = eofReader + // Reset it to a known zero reader, in case underlying one + // is unhappy being read repeatedly. + r1.Body = http_NoBody } else { r1.ContentLength = -1 r1.Body = struct { @@ -3149,113 +7330,930 @@ } } } - - if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) { + // If we're sending a non-chunked HTTP/1.1 response without a + // content-length, the only way to do that is the old HTTP/1.0 + // way, by noting the EOF with a connection close, so we need + // to set Close. + if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !http_chunked(r1.TransferEncoding) && !r1.Uncompressed { r1.Close = true } - tw, err := newTransferWriter(r1) + // Process Body,ContentLength,Close,Trailer + tw, err := http_newTransferWriter(r1) if err != nil { return err } - err = tw.WriteHeader(w) + err = tw.writeHeader(w, nil) if err != nil { return err } - err = r.Header.WriteSubset(w, respExcludeHeader) + // Rest of header + err = r.Header.WriteSubset(w, http_respExcludeHeader) if err != nil { return err } + // contentLengthAlreadySent may have been already sent for + // POST/PUT requests, even if zero length. See Issue 8180. contentLengthAlreadySent := tw.shouldSendContentLength() - if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent { + if r1.ContentLength == 0 && !http_chunked(r1.TransferEncoding) && !contentLengthAlreadySent && http_bodyAllowedForStatus(r.StatusCode) { if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil { return err } } + // End-of-header if _, err := io.WriteString(w, "\r\n"); err != nil { return err } - err = tw.WriteBody(w) + // Write body and trailer + err = tw.writeBody(w) if err != nil { return err } + // Success return nil } -// Errors introduced by the HTTP server. +func (r *http_Response) closeBody() { + if r.Body != nil { + r.Body.Close() + } +} + +// bodyIsWritable reports whether the Body supports writing. The +// Transport returns Writable bodies for 101 Switching Protocols +// responses. +// The Transport uses this method to determine whether a persistent +// connection is done being managed from its perspective. Once we +// return a writable response body to a user, the net/http package is +// done managing that connection. +func (r *http_Response) bodyIsWritable() bool { + _, ok := r.Body.(io.Writer) + return ok +} + +// isProtocolSwitch reports whether the response code and header +// indicate a successful protocol upgrade response. +func (r *http_Response) isProtocolSwitch() bool { + return http_isProtocolSwitchResponse(r.StatusCode, r.Header) +} + +// isProtocolSwitchResponse reports whether the response code and +// response header indicate a successful protocol upgrade response. +func http_isProtocolSwitchResponse(code int, h http_Header) bool { + return code == http_StatusSwitchingProtocols && http_isProtocolSwitchHeader(h) +} + +// isProtocolSwitchHeader reports whether the request or response header +// is for a protocol switch. +func http_isProtocolSwitchHeader(h http_Header) bool { + return h.Get("Upgrade") != "" && + httpguts.HeaderValuesContainsToken(h["Connection"], "Upgrade") +} + +// A ResponseController is used by an HTTP handler to control the response. +// +// A ResponseController may not be used after the [Handler.ServeHTTP] method has returned. +type http_ResponseController struct { + rw http_ResponseWriter +} + +// NewResponseController creates a [ResponseController] for a request. +// +// The ResponseWriter should be the original value passed to the [Handler.ServeHTTP] method, +// or have an Unwrap method returning the original ResponseWriter. +// +// If the ResponseWriter implements any of the following methods, the ResponseController +// will call them as appropriate: +// +// Flush() +// FlushError() error // alternative Flush returning an error +// Hijack() (net.Conn, *bufio.ReadWriter, error) +// SetReadDeadline(deadline time.Time) error +// SetWriteDeadline(deadline time.Time) error +// EnableFullDuplex() error +// +// If the ResponseWriter does not support a method, ResponseController returns +// an error matching [ErrNotSupported]. +func http_NewResponseController(rw http_ResponseWriter) *http_ResponseController { + return &http_ResponseController{rw} +} + +type http_rwUnwrapper interface { + Unwrap() http_ResponseWriter +} + +// Flush flushes buffered data to the client. +func (c *http_ResponseController) Flush() error { + rw := c.rw + for { + switch t := rw.(type) { + case interface{ FlushError() error }: + return t.FlushError() + case http_Flusher: + t.Flush() + return nil + case http_rwUnwrapper: + rw = t.Unwrap() + default: + return http_errNotSupported() + } + } +} + +// Hijack lets the caller take over the connection. +// See the [Hijacker] interface for details. +func (c *http_ResponseController) Hijack() (net.Conn, *bufio.ReadWriter, error) { + rw := c.rw + for { + switch t := rw.(type) { + case http_Hijacker: + return t.Hijack() + case http_rwUnwrapper: + rw = t.Unwrap() + default: + return nil, nil, http_errNotSupported() + } + } +} + +// SetReadDeadline sets the deadline for reading the entire request, including the body. +// Reads from the request body after the deadline has been exceeded will return an error. +// A zero value means no deadline. +// +// Setting the read deadline after it has been exceeded will not extend it. +func (c *http_ResponseController) SetReadDeadline(deadline time.Time) error { + rw := c.rw + for { + switch t := rw.(type) { + case interface{ SetReadDeadline(time.Time) error }: + return t.SetReadDeadline(deadline) + case http_rwUnwrapper: + rw = t.Unwrap() + default: + return http_errNotSupported() + } + } +} + +// SetWriteDeadline sets the deadline for writing the response. +// Writes to the response body after the deadline has been exceeded will not block, +// but may succeed if the data has been buffered. +// A zero value means no deadline. +// +// Setting the write deadline after it has been exceeded will not extend it. +func (c *http_ResponseController) SetWriteDeadline(deadline time.Time) error { + rw := c.rw + for { + switch t := rw.(type) { + case interface{ SetWriteDeadline(time.Time) error }: + return t.SetWriteDeadline(deadline) + case http_rwUnwrapper: + rw = t.Unwrap() + default: + return http_errNotSupported() + } + } +} + +// EnableFullDuplex indicates that the request handler will interleave reads from [Request.Body] +// with writes to the [ResponseWriter]. +// +// For HTTP/1 requests, the Go HTTP server by default consumes any unread portion of +// the request body before beginning to write the response, preventing handlers from +// concurrently reading from the request and writing the response. +// Calling EnableFullDuplex disables this behavior and permits handlers to continue to read +// from the request while concurrently writing the response. +// +// For HTTP/2 requests, the Go HTTP server always permits concurrent reads and responses. +func (c *http_ResponseController) EnableFullDuplex() error { + rw := c.rw + for { + switch t := rw.(type) { + case interface{ EnableFullDuplex() error }: + return t.EnableFullDuplex() + case http_rwUnwrapper: + rw = t.Unwrap() + default: + return http_errNotSupported() + } + } +} + +// errNotSupported returns an error that Is ErrNotSupported, +// but is not == to it. +func http_errNotSupported() error { + return fmt.Errorf("%w", http_ErrNotSupported) +} + +// RoundTrip should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/erda-project/erda-infra +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname badRoundTrip net/http.(*Transport).RoundTrip +func http_badRoundTrip(*http_Transport, *http_Request) (*http_Response, error) + +// RoundTrip implements the [RoundTripper] interface. +// +// For higher-level HTTP client support (such as handling of cookies +// and redirects), see [Get], [Post], and the [Client] type. +// +// Like the RoundTripper interface, the error types returned +// by RoundTrip are unspecified. +func (t *http_Transport) RoundTrip(req *http_Request) (*http_Response, error) { + if t == nil { + panic("transport is nil") + } + return t.roundTrip(req) +} + +// A routingIndex optimizes conflict detection by indexing patterns. +// +// The basic idea is to rule out patterns that cannot conflict with a given +// pattern because they have a different literal in a corresponding segment. +// See the comments in [routingIndex.possiblyConflictingPatterns] for more details. +type http_routingIndex struct { + // map from a particular segment position and value to all registered patterns + // with that value in that position. + // For example, the key {1, "b"} would hold the patterns "/a/b" and "/a/b/c" + // but not "/a", "b/a", "/a/c" or "/a/{x}". + segments map[http_routingIndexKey][]*http_pattern + // All patterns that end in a multi wildcard (including trailing slash). + // We do not try to be clever about indexing multi patterns, because there + // are unlikely to be many of them. + multis []*http_pattern +} + +type http_routingIndexKey struct { + pos int // 0-based segment position + s string // literal, or empty for wildcard +} + +func (idx *http_routingIndex) addPattern(pat *http_pattern) { + if pat.lastSegment().multi { + idx.multis = append(idx.multis, pat) + } else { + if idx.segments == nil { + idx.segments = map[http_routingIndexKey][]*http_pattern{} + } + for pos, seg := range pat.segments { + key := http_routingIndexKey{pos: pos, s: ""} + if !seg.wild { + key.s = seg.s + } + idx.segments[key] = append(idx.segments[key], pat) + } + } +} + +// possiblyConflictingPatterns calls f on all patterns that might conflict with +// pat. If f returns a non-nil error, possiblyConflictingPatterns returns immediately +// with that error. +// +// To be correct, possiblyConflictingPatterns must include all patterns that +// might conflict. But it may also include patterns that cannot conflict. +// For instance, an implementation that returns all registered patterns is correct. +// We use this fact throughout, simplifying the implementation by returning more +// patterns that we might need to. +func (idx *http_routingIndex) possiblyConflictingPatterns(pat *http_pattern, f func(*http_pattern) error) (err error) { + // Terminology: + // dollar pattern: one ending in "{$}" + // multi pattern: one ending in a trailing slash or "{x...}" wildcard + // ordinary pattern: neither of the above + + // apply f to all the pats, stopping on error. + apply := func(pats []*http_pattern) error { + if err != nil { + return err + } + for _, p := range pats { + err = f(p) + if err != nil { + return err + } + } + return nil + } + + // Our simple indexing scheme doesn't try to prune multi patterns; assume + // any of them can match the argument. + if err := apply(idx.multis); err != nil { + return err + } + if pat.lastSegment().s == "/" { + // All paths that a dollar pattern matches end in a slash; no paths that + // an ordinary pattern matches do. So only other dollar or multi + // patterns can conflict with a dollar pattern. Furthermore, conflicting + // dollar patterns must have the {$} in the same position. + return apply(idx.segments[http_routingIndexKey{s: "/", pos: len(pat.segments) - 1}]) + } + // For ordinary and multi patterns, the only conflicts can be with a multi, + // or a pattern that has the same literal or a wildcard at some literal + // position. + // We could intersect all the possible matches at each position, but we + // do something simpler: we find the position with the fewest patterns. + var lmin, wmin []*http_pattern + min := math.MaxInt + hasLit := false + for i, seg := range pat.segments { + if seg.multi { + break + } + if !seg.wild { + hasLit = true + lpats := idx.segments[http_routingIndexKey{s: seg.s, pos: i}] + wpats := idx.segments[http_routingIndexKey{s: "", pos: i}] + if sum := len(lpats) + len(wpats); sum < min { + lmin = lpats + wmin = wpats + min = sum + } + } + } + if hasLit { + apply(lmin) + apply(wmin) + return err + } + + // This pattern is all wildcards. + // Check it against everything. + for _, pats := range idx.segments { + apply(pats) + } + return err +} + +// A routingNode is a node in the decision tree. +// The same struct is used for leaf and interior nodes. +type http_routingNode struct { + // A leaf node holds a single pattern and the Handler it was registered + // with. + pattern *http_pattern + handler http_Handler + + // An interior node maps parts of the incoming request to child nodes. + // special children keys: + // "/" trailing slash (resulting from {$}) + // "" single wildcard + children http_mapping[string, *http_routingNode] + multiChild *http_routingNode // child with multi wildcard + emptyChild *http_routingNode // optimization: child with key "" +} + +// addPattern adds a pattern and its associated Handler to the tree +// at root. +func (root *http_routingNode) addPattern(p *http_pattern, h http_Handler) { + // First level of tree is host. + n := root.addChild(p.host) + // Second level of tree is method. + n = n.addChild(p.method) + // Remaining levels are path. + n.addSegments(p.segments, p, h) +} + +// addSegments adds the given segments to the tree rooted at n. +// If there are no segments, then n is a leaf node that holds +// the given pattern and handler. +func (n *http_routingNode) addSegments(segs []http_segment, p *http_pattern, h http_Handler) { + if len(segs) == 0 { + n.set(p, h) + return + } + seg := segs[0] + if seg.multi { + if len(segs) != 1 { + panic("multi wildcard not last") + } + c := &http_routingNode{} + n.multiChild = c + c.set(p, h) + } else if seg.wild { + n.addChild("").addSegments(segs[1:], p, h) + } else { + n.addChild(seg.s).addSegments(segs[1:], p, h) + } +} + +// set sets the pattern and handler for n, which +// must be a leaf node. +func (n *http_routingNode) set(p *http_pattern, h http_Handler) { + if n.pattern != nil || n.handler != nil { + panic("non-nil leaf fields") + } + n.pattern = p + n.handler = h +} + +// addChild adds a child node with the given key to n +// if one does not exist, and returns the child. +func (n *http_routingNode) addChild(key string) *http_routingNode { + if key == "" { + if n.emptyChild == nil { + n.emptyChild = &http_routingNode{} + } + return n.emptyChild + } + if c := n.findChild(key); c != nil { + return c + } + c := &http_routingNode{} + n.children.add(key, c) + return c +} + +// findChild returns the child of n with the given key, or nil +// if there is no child with that key. +func (n *http_routingNode) findChild(key string) *http_routingNode { + if key == "" { + return n.emptyChild + } + r, _ := n.children.find(key) + return r +} + +// match returns the leaf node under root that matches the arguments, and a list +// of values for pattern wildcards in the order that the wildcards appear. +// For example, if the request path is "/a/b/c" and the pattern is "/{x}/b/{y}", +// then the second return value will be []string{"a", "c"}. +func (root *http_routingNode) match(host, method, path string) (*http_routingNode, []string) { + if host != "" { + // There is a host. If there is a pattern that specifies that host and it + // matches, we are done. If the pattern doesn't match, fall through to + // try patterns with no host. + if l, m := root.findChild(host).matchMethodAndPath(method, path); l != nil { + return l, m + } + } + return root.emptyChild.matchMethodAndPath(method, path) +} + +// matchMethodAndPath matches the method and path. +// Its return values are the same as [routingNode.match]. +// The receiver should be a child of the root. +func (n *http_routingNode) matchMethodAndPath(method, path string) (*http_routingNode, []string) { + if n == nil { + return nil, nil + } + if l, m := n.findChild(method).matchPath(path, nil); l != nil { + // Exact match of method name. + return l, m + } + if method == "HEAD" { + // GET matches HEAD too. + if l, m := n.findChild("GET").matchPath(path, nil); l != nil { + return l, m + } + } + // No exact match; try patterns with no method. + return n.emptyChild.matchPath(path, nil) +} + +// matchPath matches a path. +// Its return values are the same as [routingNode.match]. +// matchPath calls itself recursively. The matches argument holds the wildcard matches +// found so far. +func (n *http_routingNode) matchPath(path string, matches []string) (*http_routingNode, []string) { + if n == nil { + return nil, nil + } + // If path is empty, then we are done. + // If n is a leaf node, we found a match; return it. + // If n is an interior node (which means it has a nil pattern), + // then we failed to match. + if path == "" { + if n.pattern == nil { + return nil, nil + } + return n, matches + } + // Get the first segment of path. + seg, rest := http_firstSegment(path) + // First try matching against patterns that have a literal for this position. + // We know by construction that such patterns are more specific than those + // with a wildcard at this position (they are either more specific, equivalent, + // or overlap, and we ruled out the first two when the patterns were registered). + if n, m := n.findChild(seg).matchPath(rest, matches); n != nil { + return n, m + } + // If matching a literal fails, try again with patterns that have a single + // wildcard (represented by an empty string in the child mapping). + // Again, by construction, patterns with a single wildcard must be more specific than + // those with a multi wildcard. + // We skip this step if the segment is a trailing slash, because single wildcards + // don't match trailing slashes. + if seg != "/" { + if n, m := n.emptyChild.matchPath(rest, append(matches, seg)); n != nil { + return n, m + } + } + // Lastly, match the pattern (there can be at most one) that has a multi + // wildcard in this position to the rest of the path. + if c := n.multiChild; c != nil { + // Don't record a match for a nameless wildcard (which arises from a + // trailing slash in the pattern). + if c.pattern.lastSegment().s != "" { + matches = append(matches, http_pathUnescape(path[1:])) // remove initial slash + } + return c, matches + } + return nil, nil +} + +// firstSegment splits path into its first segment, and the rest. +// The path must begin with "/". +// If path consists of only a slash, firstSegment returns ("/", ""). +// The segment is returned unescaped, if possible. +func http_firstSegment(path string) (seg, rest string) { + if path == "/" { + return "/", "" + } + path = path[1:] // drop initial slash + i := strings.IndexByte(path, '/') + if i < 0 { + i = len(path) + } + return http_pathUnescape(path[:i]), path[i:] +} + +// matchingMethods adds to methodSet all the methods that would result in a +// match if passed to routingNode.match with the given host and path. +func (root *http_routingNode) matchingMethods(host, path string, methodSet map[string]bool) { + if host != "" { + root.findChild(host).matchingMethodsPath(path, methodSet) + } + root.emptyChild.matchingMethodsPath(path, methodSet) + if methodSet["GET"] { + methodSet["HEAD"] = true + } +} + +func (n *http_routingNode) matchingMethodsPath(path string, set map[string]bool) { + if n == nil { + return + } + n.children.eachPair(func(method string, c *http_routingNode) bool { + if p, _ := c.matchPath(path, nil); p != nil { + set[method] = true + } + return true + }) + // Don't look at the empty child. If there were an empty + // child, it would match on any method, but we only + // call this when we fail to match on a method. +} + +var http_httpmuxgo121 = godebug.New("httpmuxgo121") + +var http_use121 bool + +// Read httpmuxgo121 once at startup, since dealing with changes to it during +// program execution is too complex and error-prone. +func init() { + if http_httpmuxgo121.Value() == "1" { + http_use121 = true + http_httpmuxgo121.IncNonDefault() + } +} + +// serveMux121 holds the state of a ServeMux needed for Go 1.21 behavior. +type http_serveMux121 struct { + mu sync.RWMutex + m map[string]http_muxEntry + es []http_muxEntry // slice of entries sorted from longest to shortest. + hosts bool // whether any patterns contain hostnames +} + +type http_muxEntry struct { + h http_Handler + pattern string +} + +// Formerly ServeMux.Handle. +func (mux *http_serveMux121) handle(pattern string, handler http_Handler) { + mux.mu.Lock() + defer mux.mu.Unlock() + + if pattern == "" { + panic("http: invalid pattern") + } + if handler == nil { + panic("http: nil handler") + } + if _, exist := mux.m[pattern]; exist { + panic("http: multiple registrations for " + pattern) + } + + if mux.m == nil { + mux.m = make(map[string]http_muxEntry) + } + e := http_muxEntry{h: handler, pattern: pattern} + mux.m[pattern] = e + if pattern[len(pattern)-1] == '/' { + mux.es = http_appendSorted(mux.es, e) + } + + if pattern[0] != '/' { + mux.hosts = true + } +} + +func http_appendSorted(es []http_muxEntry, e http_muxEntry) []http_muxEntry { + n := len(es) + i := sort.Search(n, func(i int) bool { + return len(es[i].pattern) < len(e.pattern) + }) + if i == n { + return append(es, e) + } + // we now know that i points at where we want to insert + es = append(es, http_muxEntry{}) // try to grow the slice in place, any entry works. + copy(es[i+1:], es[i:]) // Move shorter entries down + es[i] = e + return es +} + +// Formerly ServeMux.HandleFunc. +func (mux *http_serveMux121) handleFunc(pattern string, handler func(http_ResponseWriter, *http_Request)) { + if handler == nil { + panic("http: nil handler") + } + mux.handle(pattern, http_HandlerFunc(handler)) +} + +// Formerly ServeMux.Handler. +func (mux *http_serveMux121) findHandler(r *http_Request) (h http_Handler, pattern string) { + + // CONNECT requests are not canonicalized. + if r.Method == "CONNECT" { + // If r.URL.Path is /tree and its handler is not registered, + // the /tree -> /tree/ redirect applies to CONNECT requests + // but the path canonicalization does not. + if u, ok := mux.redirectToPathSlash(r.URL.Host, r.URL.Path, r.URL); ok { + return http_RedirectHandler(u.String(), http_StatusMovedPermanently), u.Path + } + + return mux.handler(r.Host, r.URL.Path) + } + + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host := http_stripHostPort(r.Host) + path := http_cleanPath(r.URL.Path) + + // If the given path is /tree and its handler is not registered, + // redirect for /tree/. + if u, ok := mux.redirectToPathSlash(host, path, r.URL); ok { + return http_RedirectHandler(u.String(), http_StatusMovedPermanently), u.Path + } + + if path != r.URL.Path { + _, pattern = mux.handler(host, path) + u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} + return http_RedirectHandler(u.String(), http_StatusMovedPermanently), pattern + } + + return mux.handler(host, r.URL.Path) +} + +// handler is the main implementation of findHandler. +// The path is known to be in canonical form, except for CONNECT methods. +func (mux *http_serveMux121) handler(host, path string) (h http_Handler, pattern string) { + mux.mu.RLock() + defer mux.mu.RUnlock() + + // Host-specific pattern takes precedence over generic ones + if mux.hosts { + h, pattern = mux.match(host + path) + } + if h == nil { + h, pattern = mux.match(path) + } + if h == nil { + h, pattern = http_NotFoundHandler(), "" + } + return +} + +// Find a handler on a handler map given a path string. +// Most-specific (longest) pattern wins. +func (mux *http_serveMux121) match(path string) (h http_Handler, pattern string) { + // Check for exact match first. + v, ok := mux.m[path] + if ok { + return v.h, v.pattern + } + + // Check for longest valid match. mux.es contains all patterns + // that end in / sorted from longest to shortest. + for _, e := range mux.es { + if strings.HasPrefix(path, e.pattern) { + return e.h, e.pattern + } + } + return nil, "" +} + +// redirectToPathSlash determines if the given path needs appending "/" to it. +// This occurs when a handler for path + "/" was already registered, but +// not for path itself. If the path needs appending to, it creates a new +// URL, setting the path to u.Path + "/" and returning true to indicate so. +func (mux *http_serveMux121) redirectToPathSlash(host, path string, u *url.URL) (*url.URL, bool) { + mux.mu.RLock() + shouldRedirect := mux.shouldRedirectRLocked(host, path) + mux.mu.RUnlock() + if !shouldRedirect { + return u, false + } + path = path + "/" + u = &url.URL{Path: path, RawQuery: u.RawQuery} + return u, true +} + +// shouldRedirectRLocked reports whether the given path and host should be redirected to +// path+"/". This should happen if a handler is registered for path+"/" but +// not path -- see comments at ServeMux. +func (mux *http_serveMux121) shouldRedirectRLocked(host, path string) bool { + p := []string{path, host + path} + + for _, c := range p { + if _, exist := mux.m[c]; exist { + return false + } + } + + n := len(path) + if n == 0 { + return false + } + for _, c := range p { + if _, exist := mux.m[c+"/"]; exist { + return path[n-1] != '/' + } + } + + return false +} + +// Errors used by the HTTP server. var ( - ErrWriteAfterFlush = errors.New("Conn.Write called after Flush") - ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") - ErrHijacked = errors.New("Conn has been hijacked") - ErrContentLength = errors.New("Conn.Write wrote more than the declared Content-Length") + // ErrBodyNotAllowed is returned by ResponseWriter.Write calls + // when the HTTP method or response code does not permit a + // body. + http_ErrBodyNotAllowed = internal.ErrBodyNotAllowed + + // ErrHijacked is returned by ResponseWriter.Write calls when + // the underlying connection has been hijacked using the + // Hijacker interface. A zero-byte write on a hijacked + // connection will return ErrHijacked without any other side + // effects. + http_ErrHijacked = errors.New("http: connection has been hijacked") + + // ErrContentLength is returned by ResponseWriter.Write calls + // when a Handler set a Content-Length response header with a + // declared size and then attempted to write more bytes than + // declared. + http_ErrContentLength = errors.New("http: wrote more than the declared Content-Length") + + // Deprecated: ErrWriteAfterFlush is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + http_ErrWriteAfterFlush = errors.New("unused") ) -// Objects implementing the Handler interface can be -// registered to serve a particular path or subtree -// in the HTTP server. +// A Handler responds to an HTTP request. // -// ServeHTTP should write reply headers and data to the ResponseWriter -// and then return. Returning signals that the request is finished -// and that the HTTP server can move on to the next request on -// the connection. +// [Handler.ServeHTTP] should write reply headers and data to the [ResponseWriter] +// and then return. Returning signals that the request is finished; it +// is not valid to use the [ResponseWriter] or read from the +// [Request.Body] after or concurrently with the completion of the +// ServeHTTP call. +// +// Depending on the HTTP client software, HTTP protocol version, and +// any intermediaries between the client and the Go server, it may not +// be possible to read from the [Request.Body] after writing to the +// [ResponseWriter]. Cautious handlers should read the [Request.Body] +// first, and then reply. +// +// Except for reading the body, handlers should not modify the +// provided Request. // // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes // that the effect of the panic was isolated to the active request. // It recovers the panic, logs a stack trace to the server error log, -// and hangs up the connection. -// -type Handler interface { - ServeHTTP(ResponseWriter, *Request) +// and either closes the network connection or sends an HTTP/2 +// RST_STREAM, depending on the HTTP protocol. To abort a handler so +// the client sees an interrupted response but the server doesn't log +// an error, panic with the value [ErrAbortHandler]. +type http_Handler interface { + ServeHTTP(http_ResponseWriter, *http_Request) } // A ResponseWriter interface is used by an HTTP handler to // construct an HTTP response. -type ResponseWriter interface { +// +// A ResponseWriter may not be used after [Handler.ServeHTTP] has returned. +type http_ResponseWriter interface { // Header returns the header map that will be sent by - // WriteHeader. Changing the header after a call to - // WriteHeader (or Write) has no effect unless the modified - // headers were declared as trailers by setting the - // "Trailer" header before the call to WriteHeader (see example). - // To suppress implicit response headers, set their value to nil. - Header() Header + // [ResponseWriter.WriteHeader]. The [Header] map also is the mechanism with which + // [Handler] implementations can set HTTP trailers. + // + // Changing the header map after a call to [ResponseWriter.WriteHeader] (or + // [ResponseWriter.Write]) has no effect unless the HTTP status code was of the + // 1xx class or the modified headers are trailers. + // + // There are two ways to set Trailers. The preferred way is to + // predeclare in the headers which trailers you will later + // send by setting the "Trailer" header to the names of the + // trailer keys which will come later. In this case, those + // keys of the Header map are treated as if they were + // trailers. See the example. The second way, for trailer + // keys not known to the [Handler] until after the first [ResponseWriter.Write], + // is to prefix the [Header] map keys with the [TrailerPrefix] + // constant value. + // + // To suppress automatic response headers (such as "Date"), set + // their value to nil. + Header() http_Header // Write writes the data to the connection as part of an HTTP reply. - // If WriteHeader has not yet been called, Write calls WriteHeader(http.StatusOK) - // before writing the data. If the Header does not contain a - // Content-Type line, Write adds a Content-Type set to the result of passing - // the initial 512 bytes of written data to DetectContentType. + // + // If [ResponseWriter.WriteHeader] has not yet been called, Write calls + // WriteHeader(http.StatusOK) before writing the data. If the Header + // does not contain a Content-Type line, Write adds a Content-Type set + // to the result of passing the initial 512 bytes of written data to + // [DetectContentType]. Additionally, if the total size of all written + // data is under a few KB and there are no Flush calls, the + // Content-Length header is added automatically. + // + // Depending on the HTTP protocol version and the client, calling + // Write or WriteHeader may prevent future reads on the + // Request.Body. For HTTP/1.x requests, handlers should read any + // needed request body data before writing the response. Once the + // headers have been flushed (due to either an explicit Flusher.Flush + // call or writing enough data to trigger a flush), the request body + // may be unavailable. For HTTP/2 requests, the Go HTTP server permits + // handlers to continue to read the request body while concurrently + // writing the response. However, such behavior may not be supported + // by all HTTP/2 clients. Handlers should read before writing if + // possible to maximize compatibility. Write([]byte) (int, error) - // WriteHeader sends an HTTP response header with status code. + // WriteHeader sends an HTTP response header with the provided + // status code. + // // If WriteHeader is not called explicitly, the first call to Write // will trigger an implicit WriteHeader(http.StatusOK). // Thus explicit calls to WriteHeader are mainly used to - // send error codes. - WriteHeader(int) + // send error codes or 1xx informational responses. + // + // The provided code must be a valid HTTP 1xx-5xx status code. + // Any number of 1xx headers may be written, followed by at most + // one 2xx-5xx header. 1xx headers are sent immediately, but 2xx-5xx + // headers may be buffered. Use the Flusher interface to send + // buffered data. The header map is cleared when 2xx-5xx headers are + // sent, but not with 1xx headers. + // + // The server will automatically send a 100 (Continue) header + // on the first read from the request body if the request has + // an "Expect: 100-continue" header. + WriteHeader(statusCode int) } // The Flusher interface is implemented by ResponseWriters that allow // an HTTP handler to flush buffered data to the client. // +// The default HTTP/1.x and HTTP/2 [ResponseWriter] implementations +// support [Flusher], but ResponseWriter wrappers may not. Handlers +// should always test for this ability at runtime. +// // Note that even for ResponseWriters that support Flush, // if the client is connected through an HTTP proxy, // the buffered data may not reach the client until the response // completes. -type Flusher interface { +type http_Flusher interface { // Flush sends any buffered data to the client. Flush() } // The Hijacker interface is implemented by ResponseWriters that allow // an HTTP handler to take over the connection. -type Hijacker interface { +// +// The default [ResponseWriter] for HTTP/1.x connections supports +// Hijacker, but HTTP/2 connections intentionally do not. +// ResponseWriter wrappers may also not support Hijacker. Handlers +// should always test for this ability at runtime. +type http_Hijacker interface { // Hijack lets the caller take over the connection. - // After a call to Hijack(), the HTTP server library + // After a call to Hijack the HTTP server library // will not do anything else with the connection. // // It becomes the caller's responsibility to manage @@ -3265,6 +8263,14 @@ // already set, depending on the configuration of the // Server. It is the caller's responsibility to set // or clear those deadlines as needed. + // + // The returned bufio.Reader may contain unprocessed buffered + // data from the client. + // + // After a call to Hijack, the original Request.Body must not + // be used. The original Request's Context remains valid and + // is not canceled until the Request's ServeHTTP method + // returns. Hijack() (net.Conn, *bufio.ReadWriter, error) } @@ -3273,117 +8279,137 @@ // // This mechanism can be used to cancel long operations on the server // if the client has disconnected before the response is ready. -type CloseNotifier interface { - // CloseNotify returns a channel that receives a single value - // when the client connection has gone away. +// +// Deprecated: the CloseNotifier interface predates Go's context package. +// New code should use [Request.Context] instead. +type http_CloseNotifier interface { + // CloseNotify returns a channel that receives at most a + // single value (true) when the client connection has gone + // away. + // + // CloseNotify may wait to notify until Request.Body has been + // fully read. + // + // After the Handler has returned, there is no guarantee + // that the channel receives a value. + // + // If the protocol is HTTP/1.1 and CloseNotify is called while + // processing an idempotent request (such as GET) while + // HTTP/1.1 pipelining is in use, the arrival of a subsequent + // pipelined request may cause a value to be sent on the + // returned channel. In practice HTTP/1.1 pipelining is not + // enabled in browsers and not seen often in the wild. If this + // is a problem, use HTTP/2 or only use CloseNotify on methods + // such as POST. CloseNotify() <-chan bool } -// A conn represents the server side of an HTTP connection. -type conn struct { - remoteAddr string // network address of remote side - server *Server // the Server on which the connection arrived - rwc net.Conn // i/o connection - w io.Writer // checkConnErrorWriter's copy of wrc, not zeroed on Hijack - werr error // any errors writing to w - sr liveSwitchReader // where the LimitReader reads from; usually the rwc - lr *io.LimitedReader // io.LimitReader(sr) - buf *bufio.ReadWriter // buffered(lr,rwc), reading from bufio->limitReader->sr->rwc - tlsState *tls.ConnectionState // or nil when not using TLS - lastMethod string // method of previous request, or "" +var ( + // ServerContextKey is a context key. It can be used in HTTP + // handlers with Context.Value to access the server that + // started the handler. The associated value will be of + // type *Server. + http_ServerContextKey = &http_contextKey{"http-server"} - mu sync.Mutex // guards the following - clientGone bool // if client has disconnected mid-request - closeNotifyc chan bool // made lazily - hijackedv bool // connection has been hijacked by handler + // LocalAddrContextKey is a context key. It can be used in + // HTTP handlers with Context.Value to access the local + // address the connection arrived on. + // The associated value will be of type net.Addr. + http_LocalAddrContextKey = &http_contextKey{"local-addr"} +) + +// A conn represents the server side of an HTTP connection. +type http_conn struct { + // server is the server on which the connection arrived. + // Immutable; never nil. + server *http_Server + + // cancelCtx cancels the connection-level context. + cancelCtx context.CancelFunc + + // rwc is the underlying network connection. + // This is never wrapped by other types and is the value given out + // to [Hijacker] callers. It is usually of type *net.TCPConn or + // *tls.Conn. + rwc net.Conn + + // remoteAddr is rwc.RemoteAddr().String(). It is not populated synchronously + // inside the Listener's Accept goroutine, as some implementations block. + // It is populated immediately inside the (*conn).serve goroutine. + // This is the value of a Handler's (*Request).RemoteAddr. + remoteAddr string + + // tlsState is the TLS connection state when using TLS. + // nil means not TLS. + tlsState *tls.ConnectionState + + // werr is set to the first write error to rwc. + // It is set via checkConnErrorWriter{w}, where bufw writes. + werr error + + // r is bufr's read source. It's a wrapper around rwc that provides + // io.LimitedReader-style limiting (while reading request headers) + // and functionality to support CloseNotifier. See *connReader docs. + r *http_connReader + + // bufr reads from r. + bufr *bufio.Reader + + // bufw writes to checkConnErrorWriter{c}, which populates werr on error. + bufw *bufio.Writer + + // lastMethod is the method of the most recent request + // on this connection, if any. + lastMethod string + + curReq atomic.Pointer[http_response] // (which has a Request in it) + + curState atomic.Uint64 // packed (unixtime<<8|uint8(ConnState)) + + // mu guards hijackedv + mu sync.Mutex + + // hijackedv is whether this connection has been hijacked + // by a Handler with the Hijacker interface. + // It is guarded by mu. + hijackedv bool } -func (c *conn) hijacked() bool { +func (c *http_conn) hijacked() bool { c.mu.Lock() defer c.mu.Unlock() return c.hijackedv } -func (c *conn) hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { - c.mu.Lock() - defer c.mu.Unlock() +// c.mu must be held. +func (c *http_conn) hijackLocked() (rwc net.Conn, buf *bufio.ReadWriter, err error) { if c.hijackedv { - return nil, nil, ErrHijacked + return nil, nil, http_ErrHijacked } - if c.closeNotifyc != nil { - return nil, nil, errors.New("http: Hijack is incompatible with use of CloseNotifier") - } + c.r.abortPendingRead() + c.hijackedv = true rwc = c.rwc - buf = c.buf - c.rwc = nil - c.buf = nil - c.setState(rwc, StateHijacked) - return -} + rwc.SetDeadline(time.Time{}) -func (c *conn) closeNotify() <-chan bool { - c.mu.Lock() - defer c.mu.Unlock() - if c.closeNotifyc == nil { - c.closeNotifyc = make(chan bool, 1) - if c.hijackedv { - - return c.closeNotifyc + if c.r.hasByte { + if _, err := c.bufr.Peek(c.bufr.Buffered() + 1); err != nil { + return nil, nil, fmt.Errorf("unexpected Peek failure reading buffered byte: %v", err) } - pr, pw := io.Pipe() - - readSource := c.sr.r - c.sr.Lock() - c.sr.r = pr - c.sr.Unlock() - go func() { - _, err := io.Copy(pw, readSource) - if err == nil { - err = io.EOF - } - pw.CloseWithError(err) - c.noteClientGone() - }() } - return c.closeNotifyc -} + c.bufw.Reset(rwc) + buf = bufio.NewReadWriter(c.bufr, c.bufw) -func (c *conn) noteClientGone() { - c.mu.Lock() - defer c.mu.Unlock() - if c.closeNotifyc != nil && !c.clientGone { - c.closeNotifyc <- true - } - c.clientGone = true -} - -// A switchWriter can have its Writer changed at runtime. -// It's not safe for concurrent Writes and switches. -type switchWriter struct { - io.Writer -} - -// A liveSwitchReader can have its Reader changed at runtime. It's -// safe for concurrent reads and switches, if its mutex is held. -type liveSwitchReader struct { - sync.Mutex - r io.Reader -} - -func (sr *liveSwitchReader) Read(p []byte) (n int, err error) { - sr.Lock() - r := sr.r - sr.Unlock() - return r.Read(p) + c.setState(rwc, http_StateHijacked, http_runHooks) + return } // This should be >= 512 bytes for DetectContentType, // but otherwise it's somewhat arbitrary. -const bufferBeforeChunkingSize = 2048 +const http_bufferBeforeChunkingSize = 2048 // chunkWriter writes to a response's conn buffer, and is the writer -// wrapped by the response.bufw buffered writer. +// wrapped by the response.w buffered writer. // // chunkWriter also is responsible for finalizing the Header, including // conditionally setting the Content-Type and setting a Content-Length @@ -3391,14 +8417,14 @@ // size. It also conditionally adds chunk headers, when in chunking mode. // // See the comment above (*response).Write for the entire write flow. -type chunkWriter struct { - res *response +type http_chunkWriter struct { + res *http_response // header is either nil or a deep clone of res.handlerHeader - // at the time of res.WriteHeader, if res.WriteHeader is + // at the time of res.writeHeader, if res.writeHeader is // called and extra buffering is being done to calculate // Content-Type and/or Content-Length. - header Header + header http_Header // wroteHeader tells whether the header's been written to "the // wire" (or rather: w.conn.buf). this is unlike @@ -3411,28 +8437,28 @@ } var ( - crlf = []byte("\r\n") - colonSpace = []byte(": ") + http_crlf = []byte("\r\n") + http_colonSpace = []byte(": ") ) -func (cw *chunkWriter) Write(p []byte) (n int, err error) { +func (cw *http_chunkWriter) Write(p []byte) (n int, err error) { if !cw.wroteHeader { cw.writeHeader(p) } if cw.res.req.Method == "HEAD" { - + // Eat writes. return len(p), nil } if cw.chunking { - _, err = fmt.Fprintf(cw.res.conn.buf, "%x\r\n", len(p)) + _, err = fmt.Fprintf(cw.res.conn.bufw, "%x\r\n", len(p)) if err != nil { cw.res.conn.rwc.Close() return } } - n, err = cw.res.conn.buf.Write(p) + n, err = cw.res.conn.bufw.Write(p) if cw.chunking && err == nil { - _, err = cw.res.conn.buf.Write(crlf) + _, err = cw.res.conn.bufw.Write(http_crlf) } if err != nil { cw.res.conn.rwc.Close() @@ -3440,51 +8466,59 @@ return } -func (cw *chunkWriter) flush() { +func (cw *http_chunkWriter) flush() error { if !cw.wroteHeader { cw.writeHeader(nil) } - cw.res.conn.buf.Flush() + return cw.res.conn.bufw.Flush() } -func (cw *chunkWriter) close() { +func (cw *http_chunkWriter) close() { if !cw.wroteHeader { cw.writeHeader(nil) } if cw.chunking { - bw := cw.res.conn.buf - + bw := cw.res.conn.bufw // conn's bufio writer + // zero chunk to mark EOF bw.WriteString("0\r\n") - if len(cw.res.trailers) > 0 { - trailers := make(Header) - for _, h := range cw.res.trailers { - if vv := cw.res.handlerHeader[h]; len(vv) > 0 { - trailers[h] = vv - } - } - trailers.Write(bw) + if trailers := cw.res.finalTrailers(); trailers != nil { + trailers.Write(bw) // the writer handles noting errors } - + // final blank line after the trailers (whether + // present or not) bw.WriteString("\r\n") } } // A response represents the server side of an HTTP response. -type response struct { - conn *conn - req *Request // request for this response - wroteHeader bool // reply header has been (logically) written - wroteContinue bool // 100 Continue response was written +type http_response struct { + conn *http_conn + req *http_Request // request for this response + reqBody *http_body // nil when NoBody + cancelCtx context.CancelFunc // when ServeHTTP exits + wroteHeader bool // a non-1xx header has been (logically) written + wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" + wantsClose bool // HTTP request has Connection "close" + ecReader *http_expectContinueReader + + // canWriteContinue is an atomic boolean that says whether or + // not a 100 Continue header can be written to the + // connection. + // writeContinueMu must be held while writing the header. + // These two fields together synchronize the body reader (the + // expectContinueReader, which wants to write 100 Continue) + // against the main writer. + writeContinueMu sync.Mutex + canWriteContinue atomic.Bool w *bufio.Writer // buffers output in chunks to chunkWriter - cw chunkWriter - sw *switchWriter // of the bufio.Writer, for return to putBufioWriter + cw http_chunkWriter // handlerHeader is the Header that Handlers get access to, // which may be retained and mutated even after WriteHeader. // handlerHeader is copied into cw.header at WriteHeader // time, and privately mutated thereafter. - handlerHeader Header + handlerHeader http_Header calledHeader bool // handler accessed handlerHeader via Header written int64 // number of bytes written in body @@ -3497,6 +8531,10 @@ // Content-Length. closeAfterReply bool + // When fullDuplex is false (the default), we consume any remaining + // request body before starting to write a response. + fullDuplex bool + // requestBodyLimitHit is set by requestTooLarge when // maxBytesReader hits its max size. It is checked in // WriteHeader, to make sure we don't consume the @@ -3507,26 +8545,84 @@ requestBodyLimitHit bool // trailers are the headers to be sent after the handler - // finishes writing the body. This field is initialized from + // finishes writing the body. This field is initialized from // the Trailer response header when the response header is // written. trailers []string - handlerDone bool // set true when the handler exits + handlerDone atomic.Bool // set true when the handler exits - // Buffers for Date and Content-Length - dateBuf [len(TimeFormat)]byte - clenBuf [10]byte + // Buffers for Date, Content-Length, and status code + dateBuf [len(http_TimeFormat)]byte + clenBuf [10]byte + statusBuf [3]byte + + // lazyCloseNotifyMu protects closeNotifyCh and closeNotifyTriggered. + lazyCloseNotifyMu sync.Mutex + // closeNotifyCh is the channel returned by CloseNotify. + closeNotifyCh chan bool + // closeNotifyTriggered tracks prior closeNotify calls. + closeNotifyTriggered bool +} + +func (c *http_response) SetReadDeadline(deadline time.Time) error { + return c.conn.rwc.SetReadDeadline(deadline) +} + +func (c *http_response) SetWriteDeadline(deadline time.Time) error { + return c.conn.rwc.SetWriteDeadline(deadline) +} + +func (c *http_response) EnableFullDuplex() error { + c.fullDuplex = true + return nil +} + +// TrailerPrefix is a magic prefix for [ResponseWriter.Header] map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// +// https://pkg.go.dev/net/http#ResponseWriter +// https://pkg.go.dev/net/http#example-ResponseWriter-Trailers +const http_TrailerPrefix = "Trailer:" + +// finalTrailers is called after the Handler exits and returns a non-nil +// value if the Handler set any trailers. +func (w *http_response) finalTrailers() http_Header { + var t http_Header + for k, vv := range w.handlerHeader { + if kk, found := strings.CutPrefix(k, http_TrailerPrefix); found { + if t == nil { + t = make(http_Header) + } + t[kk] = vv + } + } + for _, k := range w.trailers { + if t == nil { + t = make(http_Header) + } + for _, v := range w.handlerHeader[k] { + t.Add(k, v) + } + } + return t } // declareTrailer is called for each Trailer header when the // response header is written. It notes that a header will need to be // written in the trailers at the end of the response. -func (w *response) declareTrailer(k string) { - k = CanonicalHeaderKey(k) - switch k { - case "Transfer-Encoding", "Content-Length", "Trailer": - +func (w *http_response) declareTrailer(k string) { + k = http_CanonicalHeaderKey(k) + if !httpguts.ValidTrailerHeader(k) { + // Forbidden by RFC 7230, section 4.1.2 return } w.trailers = append(w.trailers, k) @@ -3534,7 +8630,7 @@ // requestTooLarge is called by maxBytesReader when too much input has // been read from the client. -func (w *response) requestTooLarge() { +func (w *http_response) requestTooLarge() { w.closeAfterReply = true w.requestBodyLimitHit = true if !w.wroteHeader { @@ -3542,131 +8638,353 @@ } } -// needsSniff reports whether a Content-Type still needs to be sniffed. -func (w *response) needsSniff() bool { - _, haveType := w.handlerHeader["Content-Type"] - return !w.cw.wroteHeader && !haveType && w.written < sniffLen +// disableWriteContinue stops Request.Body.Read from sending an automatic +// 100 Continue. As the name implies, it is only useful when the request +// expects a 100 Continue and the body is wrapped in an expectContinueReader; +// otherwise, it is a no-op. +// If a 100-Continue is being written, it waits for it to complete before +// continuing. If skipDrain is true, it also prevents the server from draining +// the request body and flags the connection to be closed after the reply, as +// the client will never send the body. +func (w *http_response) disableWriteContinue(skipDrain bool) { + if w.ecReader == nil { + return + } + w.writeContinueMu.Lock() + if w.canWriteContinue.Load() { + w.canWriteContinue.Store(false) + if skipDrain { + // Make sure that the connection will not be reused by sending + // "Connection: close" header in the response. + w.closeAfterReply = true + // Ensure that the body will not be drained in Close. + w.ecReader.closed.Store(true) + } + } + w.writeContinueMu.Unlock() } // writerOnly hides an io.Writer value's optional ReadFrom method // from io.Copy. -type writerOnly struct { +type http_writerOnly struct { io.Writer } -func srcIsRegularFile(src io.Reader) (isRegular bool, err error) { - switch v := src.(type) { - case *os.File: - fi, err := v.Stat() - if err != nil { - return false, err - } - return fi.Mode().IsRegular(), nil - case *io.LimitedReader: - return srcIsRegularFile(v.R) - default: - return - } -} +// ReadFrom is here to optimize copying from an [*os.File] regular file +// to a [*net.TCPConn] with sendfile, or from a supported src type such +// as a *net.TCPConn on Linux with splice. +func (w *http_response) ReadFrom(src io.Reader) (n int64, err error) { + buf := http_getCopyBuf() + defer http_putCopyBuf(buf) -// ReadFrom is here to optimize copying from an *os.File regular file -// to a *net.TCPConn with sendfile. -func (w *response) ReadFrom(src io.Reader) (n int64, err error) { - + // Our underlying w.conn.rwc is usually a *TCPConn (with its + // own ReadFrom method). If not, just fall back to the normal + // copy method. rf, ok := w.conn.rwc.(io.ReaderFrom) - regFile, err := srcIsRegularFile(src) - if err != nil { - return 0, err - } - if !ok || !regFile { - return io.Copy(writerOnly{w}, src) + if !ok { + return io.CopyBuffer(http_writerOnly{w}, src, buf) } - if !w.wroteHeader { - w.WriteHeader(StatusOK) - } - - if w.needsSniff() { - n0, err := io.Copy(writerOnly{w}, io.LimitReader(src, sniffLen)) + // Copy the first sniffLen bytes before switching to ReadFrom. + // This ensures we don't start writing the response before the + // source is available (see golang.org/issue/5660) and provides + // enough bytes to perform Content-Type sniffing when required. + if !w.cw.wroteHeader { + n0, err := io.CopyBuffer(http_writerOnly{w}, io.LimitReader(src, internal.SniffLen), buf) n += n0 - if err != nil { + if err != nil || n0 < internal.SniffLen { return n, err } } - w.w.Flush() - w.cw.flush() + w.w.Flush() // get rid of any previous writes + w.cw.flush() // make sure Header is written; flush data to rwc - if !w.cw.chunking && w.bodyAllowed() { + // Now that cw has been flushed, its chunking field is guaranteed initialized. + if !w.cw.chunking && w.bodyAllowed() && w.req.Method != "HEAD" { + // When a content length is declared, but exceeded; any excess bytes + // from src should be ignored, and ErrContentLength should be returned. + // This mirrors the behavior of response.Write. + if w.contentLength != -1 { + defer func(originalReader io.Reader) { + if w.written != w.contentLength { + return + } + if n, _ := originalReader.Read([]byte{0}); err == nil && n != 0 { + err = http_ErrContentLength + } + }(src) + // src can be an io.LimitedReader already. To avoid unnecessary + // alloc and having to unnest readers repeatedly in net.sendFile, + // just adjust the existing LimitedReader N when this is the case. + if lr, ok := src.(*io.LimitedReader); ok { + if lenDiff := lr.N - (w.contentLength - w.written); lenDiff > 0 { + defer func() { lr.N += lenDiff }() + lr.N -= lenDiff + } + } else { + src = io.LimitReader(src, w.contentLength-w.written) + } + } n0, err := rf.ReadFrom(src) n += n0 w.written += n0 return n, err } - n0, err := io.Copy(writerOnly{w}, src) + n0, err := io.CopyBuffer(http_writerOnly{w}, src, buf) n += n0 return n, err } -// noLimit is an effective infinite upper bound for io.LimitedReader -const noLimit int64 = (1 << 63) - 1 - // debugServerConnections controls whether all server connections are wrapped // with a verbose logging wrapper. -const debugServerConnections = false +const http_debugServerConnections = false // Create new connection from rwc. -func (srv *Server) newConn(rwc net.Conn) (c *conn, err error) { - c = new(conn) - c.remoteAddr = rwc.RemoteAddr().String() - c.server = srv - c.rwc = rwc - c.w = rwc - if debugServerConnections { - c.rwc = newLoggingConn("server", c.rwc) +func (s *http_Server) newConn(rwc net.Conn) *http_conn { + c := &http_conn{ + server: s, + rwc: rwc, } - c.sr.r = c.rwc - c.lr = io.LimitReader(&c.sr, noLimit).(*io.LimitedReader) - br := newBufioReader(c.lr) - bw := newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) - c.buf = bufio.NewReadWriter(br, bw) - return c, nil + if http_debugServerConnections { + c.rwc = http_newLoggingConn("server", c.rwc) + } + return c +} + +type http_readResult struct { + _ http_incomparable + n int + err error + b byte // byte read, if n == 1 +} + +// connReader is the io.Reader wrapper used by *conn. It combines a +// selectively-activated io.LimitedReader (to bound request header +// read sizes) with support for selectively keeping an io.Reader.Read +// call blocked in a background goroutine to wait for activity and +// trigger a CloseNotifier channel. +// After a Handler has hijacked the conn and exited, connReader behaves like a +// proxy for the net.Conn and the aforementioned behavior is bypassed. +type http_connReader struct { + rwc net.Conn // rwc is the underlying network connection. + + mu sync.Mutex // guards following + conn *http_conn // conn is nil after handler exit. + hasByte bool + byteBuf [1]byte + cond *sync.Cond + inRead bool + aborted bool // set true before conn.rwc deadline is set to past + remain int64 // bytes remaining +} + +func (cr *http_connReader) lock() { + cr.mu.Lock() + if cr.cond == nil { + cr.cond = sync.NewCond(&cr.mu) + } +} + +func (cr *http_connReader) unlock() { cr.mu.Unlock() } + +func (cr *http_connReader) releaseConn() { + cr.lock() + defer cr.unlock() + cr.conn = nil +} + +func (cr *http_connReader) startBackgroundRead() { + cr.lock() + defer cr.unlock() + if cr.inRead { + panic("invalid concurrent Body.Read call") + } + if cr.hasByte { + return + } + cr.inRead = true + cr.rwc.SetReadDeadline(time.Time{}) + go cr.backgroundRead() +} + +func (cr *http_connReader) backgroundRead() { + n, err := cr.rwc.Read(cr.byteBuf[:]) + cr.lock() + if n == 1 { + cr.hasByte = true + // We were past the end of the previous request's body already + // (since we wouldn't be in a background read otherwise), so + // this is a pipelined HTTP request. Prior to Go 1.11 we used to + // send on the CloseNotify channel and cancel the context here, + // but the behavior was documented as only "may", and we only + // did that because that's how CloseNotify accidentally behaved + // in very early Go releases prior to context support. Once we + // added context support, people used a Handler's + // Request.Context() and passed it along. Having that context + // cancel on pipelined HTTP requests caused problems. + // Fortunately, almost nothing uses HTTP/1.x pipelining. + // Unfortunately, apt-get does, or sometimes does. + // New Go 1.11 behavior: don't fire CloseNotify or cancel + // contexts on pipelined requests. Shouldn't affect people, but + // fixes cases like Issue 23921. This does mean that a client + // closing their TCP connection after sending a pipelined + // request won't cancel the context, but we'll catch that on any + // write failure (in checkConnErrorWriter.Write). + // If the server never writes, yes, there are still contrived + // server & client behaviors where this fails to ever cancel the + // context, but that's kinda why HTTP/1.x pipelining died + // anyway. + } + if ne, ok := err.(net.Error); ok && cr.aborted && ne.Timeout() { + // Ignore this error. It's the expected error from + // another goroutine calling abortPendingRead. + } else if err != nil { + cr.handleReadErrorLocked(err) + } + cr.aborted = false + cr.inRead = false + cr.unlock() + cr.cond.Broadcast() +} + +func (cr *http_connReader) abortPendingRead() { + cr.lock() + defer cr.unlock() + if !cr.inRead { + return + } + cr.aborted = true + cr.rwc.SetReadDeadline(http_aLongTimeAgo) + for cr.inRead { + cr.cond.Wait() + } + cr.rwc.SetReadDeadline(time.Time{}) +} + +func (cr *http_connReader) setReadLimit(remain int64) { cr.remain = remain } + +func (cr *http_connReader) setInfiniteReadLimit() { cr.remain = http_maxInt64 } + +func (cr *http_connReader) hitReadLimit() bool { return cr.remain <= 0 } + +// handleReadErrorLocked is called whenever a Read from the client returns a +// non-nil error. +// +// The provided non-nil err is almost always io.EOF or a "use of +// closed network connection". In any case, the error is not +// particularly interesting, except perhaps for debugging during +// development. Any error means the connection is dead and we should +// down its context. +// +// The caller must hold connReader.mu. +func (cr *http_connReader) handleReadErrorLocked(_ error) { + if cr.conn == nil { + return + } + cr.conn.cancelCtx() + if res := cr.conn.curReq.Load(); res != nil { + res.closeNotify() + } +} + +func (cr *http_connReader) Read(p []byte) (n int, err error) { + cr.lock() + if cr.conn == nil { + cr.unlock() + return cr.rwc.Read(p) + } + if cr.inRead { + hijacked := cr.conn.hijacked() + cr.unlock() + if hijacked { + panic("invalid Body.Read call. After hijacked, the original Request must not be used") + } + panic("invalid concurrent Body.Read call") + } + if cr.hitReadLimit() { + cr.unlock() + return 0, io.EOF + } + if len(p) == 0 { + cr.unlock() + return 0, nil + } + if int64(len(p)) > cr.remain { + p = p[:cr.remain] + } + if cr.hasByte { + p[0] = cr.byteBuf[0] + cr.hasByte = false + cr.unlock() + return 1, nil + } + cr.inRead = true + cr.unlock() + n, err = cr.rwc.Read(p) + + cr.lock() + cr.inRead = false + if err != nil { + cr.handleReadErrorLocked(err) + } + cr.remain -= int64(n) + cr.unlock() + + cr.cond.Broadcast() + return n, err } var ( - bufioReaderPool sync.Pool - bufioWriter2kPool sync.Pool - bufioWriter4kPool sync.Pool + http_bufioReaderPool sync.Pool + http_bufioWriter2kPool sync.Pool + http_bufioWriter4kPool sync.Pool ) -func bufioWriterPool(size int) *sync.Pool { +const http_copyBufPoolSize = 32 * 1024 + +var http_copyBufPool = sync.Pool{New: func() any { return new([http_copyBufPoolSize]byte) }} + +func http_getCopyBuf() []byte { + return http_copyBufPool.Get().(*[http_copyBufPoolSize]byte)[:] +} + +func http_putCopyBuf(b []byte) { + if len(b) != http_copyBufPoolSize { + panic("trying to put back buffer of the wrong size in the copyBufPool") + } + http_copyBufPool.Put((*[http_copyBufPoolSize]byte)(b)) +} + +func http_bufioWriterPool(size int) *sync.Pool { switch size { case 2 << 10: - return &bufioWriter2kPool + return &http_bufioWriter2kPool case 4 << 10: - return &bufioWriter4kPool + return &http_bufioWriter4kPool } return nil } -func newBufioReader(r io.Reader) *bufio.Reader { - if v := bufioReaderPool.Get(); v != nil { +func http_newBufioReader(r io.Reader) *bufio.Reader { + if v := http_bufioReaderPool.Get(); v != nil { br := v.(*bufio.Reader) br.Reset(r) return br } - + // Note: if this reader size is ever changed, update + // TestHandlerBodyClose's assumptions. return bufio.NewReader(r) } -func putBufioReader(br *bufio.Reader) { +func http_putBufioReader(br *bufio.Reader) { br.Reset(nil) - bufioReaderPool.Put(br) + http_bufioReaderPool.Put(br) } -func newBufioWriterSize(w io.Writer, size int) *bufio.Writer { - pool := bufioWriterPool(size) +func http_newBufioWriterSize(w io.Writer, size int) *bufio.Writer { + pool := http_bufioWriterPool(size) if pool != nil { if v := pool.Get(); v != nil { bw := v.(*bufio.Writer) @@ -3677,141 +8995,227 @@ return bufio.NewWriterSize(w, size) } -func putBufioWriter(bw *bufio.Writer) { +func http_putBufioWriter(bw *bufio.Writer) { bw.Reset(nil) - if pool := bufioWriterPool(bw.Available()); pool != nil { + if pool := http_bufioWriterPool(bw.Available()); pool != nil { pool.Put(bw) } } // DefaultMaxHeaderBytes is the maximum permitted size of the headers // in an HTTP request. -// This can be overridden by setting Server.MaxHeaderBytes. -const DefaultMaxHeaderBytes = 1 << 20 // 1 MB +// This can be overridden by setting [Server.MaxHeaderBytes]. +const http_DefaultMaxHeaderBytes = 1 << 20 // 1 MB -func (srv *Server) maxHeaderBytes() int { - if srv.MaxHeaderBytes > 0 { - return srv.MaxHeaderBytes +// DefaultMaxHeaderValueCount is the maximum permitted number of +// header values in an HTTP request. +// This can be overridden by setting [Server.MaxHeaderValueCount]. +const http_DefaultMaxHeaderValueCount = 500 + +func (s *http_Server) maxHeaderBytes() int { + if s.MaxHeaderBytes > 0 { + return s.MaxHeaderBytes } - return DefaultMaxHeaderBytes + return http_DefaultMaxHeaderBytes } -func (srv *Server) initialLimitedReaderSize() int64 { - return int64(srv.maxHeaderBytes()) + 4096 +func (s *http_Server) maxHeaderValueCount() int { + if s.MaxHeaderValueCount > 0 { + return s.MaxHeaderValueCount + } + return http_DefaultMaxHeaderValueCount } -// wrapper around io.ReaderCloser which on first read, sends an +func (s *http_Server) initialReadLimitSize() int64 { + return int64(s.maxHeaderBytes()) + 4096 // bufio slop +} + +// tlsHandshakeTimeout returns the time limit permitted for the TLS +// handshake, or zero for unlimited. +// +// It returns the minimum of any positive ReadHeaderTimeout, +// ReadTimeout, or WriteTimeout. +func (s *http_Server) tlsHandshakeTimeout() time.Duration { + var ret time.Duration + for _, v := range [...]time.Duration{ + s.ReadHeaderTimeout, + s.ReadTimeout, + s.WriteTimeout, + } { + if v <= 0 { + continue + } + if ret == 0 || v < ret { + ret = v + } + } + return ret +} + +// wrapper around io.ReadCloser which on first read, sends an // HTTP/1.1 100 Continue header -type expectContinueReader struct { - resp *response +type http_expectContinueReader struct { + resp *http_response readCloser io.ReadCloser - closed bool - sawEOF bool + closed atomic.Bool } -func (ecr *expectContinueReader) Read(p []byte) (n int, err error) { - if ecr.closed { - return 0, ErrBodyReadAfterClose +func (ecr *http_expectContinueReader) Read(p []byte) (n int, err error) { + if ecr.closed.Load() { + return 0, http_ErrBodyReadAfterClose } - if !ecr.resp.wroteContinue && !ecr.resp.conn.hijacked() { - ecr.resp.wroteContinue = true - ecr.resp.conn.buf.WriteString("HTTP/1.1 100 Continue\r\n\r\n") - ecr.resp.conn.buf.Flush() + w := ecr.resp + if w.canWriteContinue.Load() { + w.writeContinueMu.Lock() + if w.canWriteContinue.Load() { + w.conn.bufw.WriteString("HTTP/1.1 100 Continue\r\n\r\n") + w.conn.bufw.Flush() + w.canWriteContinue.Store(false) + } + w.writeContinueMu.Unlock() } - n, err = ecr.readCloser.Read(p) - if err == io.EOF { - ecr.sawEOF = true - } - return + return ecr.readCloser.Read(p) } -func (ecr *expectContinueReader) Close() error { - ecr.closed = true +func (ecr *http_expectContinueReader) Close() error { + if ecr.resp.canWriteContinue.Load() { + ecr.resp.disableWriteContinue(true) + } + if ecr.closed.Swap(true) { + return nil + } return ecr.readCloser.Close() } -// TimeFormat is the time format to use with -// time.Parse and time.Time.Format when parsing -// or generating times in HTTP headers. -// It is like time.RFC1123 but hard codes GMT as the time zone. -const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" +// TimeFormat is the time format to use when generating times in HTTP +// headers. It is like [time.RFC1123] but hard-codes GMT as the time +// zone. The time being formatted must be in UTC for Format to +// generate the correct format. +// +// For parsing this time format, see [ParseTime]. +const http_TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" -// appendTime is a non-allocating version of []byte(t.UTC().Format(TimeFormat)) -func appendTime(b []byte, t time.Time) []byte { - const days = "SunMonTueWedThuFriSat" - const months = "JanFebMarAprMayJunJulAugSepOctNovDec" - - t = t.UTC() - yy, mm, dd := t.Date() - hh, mn, ss := t.Clock() - day := days[3*t.Weekday():] - mon := months[3*(mm-1):] - - return append(b, - day[0], day[1], day[2], ',', ' ', - byte('0'+dd/10), byte('0'+dd%10), ' ', - mon[0], mon[1], mon[2], ' ', - byte('0'+yy/1000), byte('0'+(yy/100)%10), byte('0'+(yy/10)%10), byte('0'+yy%10), ' ', - byte('0'+hh/10), byte('0'+hh%10), ':', - byte('0'+mn/10), byte('0'+mn%10), ':', - byte('0'+ss/10), byte('0'+ss%10), ' ', - 'G', 'M', 'T') -} - -var errTooLarge = errors.New("http: request too large") +var http_errTooLarge = errors.New("http: request too large") // Read next request from connection. -func (c *conn) readRequest() (w *response, err error) { +func (c *http_conn) readRequest(ctx context.Context) (w *http_response, err error) { if c.hijacked() { - return nil, ErrHijacked + return nil, http_ErrHijacked } - if d := c.server.ReadTimeout; d != 0 { - c.rwc.SetReadDeadline(time.Now().Add(d)) + t0 := time.Now() + var wholeReqDeadline time.Time // or zero if none + if d := c.server.ReadTimeout; d > 0 { + wholeReqDeadline = t0.Add(d) } - if d := c.server.WriteTimeout; d != 0 { + if d := c.server.WriteTimeout; d > 0 { defer func() { c.rwc.SetWriteDeadline(time.Now().Add(d)) }() } - c.lr.N = c.server.initialLimitedReaderSize() + c.r.setReadLimit(c.server.initialReadLimitSize()) if c.lastMethod == "POST" { - - peek, _ := c.buf.Reader.Peek(4) - c.buf.Reader.Discard(numLeadingCRorLF(peek)) + // RFC 7230 section 3 tolerance for old buggy clients. + peek, _ := c.bufr.Peek(4) // ReadRequest will get err below + c.bufr.Discard(http_numLeadingCRorLF(peek)) } - var req *Request - if req, err = ReadRequest(c.buf.Reader); err != nil { - if c.lr.N == 0 { - return nil, errTooLarge + req, err := http_readRequestLimit(c.bufr, int64(c.server.maxHeaderValueCount())) + if err != nil { + if c.r.hitReadLimit() { + return nil, http_errTooLarge } return nil, err } - c.lr.N = noLimit - c.lastMethod = req.Method + if !http_http1ServerSupportsRequest(req) { + return nil, http_statusError{http_StatusHTTPVersionNotSupported, "unsupported protocol version"} + } + + c.lastMethod = req.Method + c.r.setInfiniteReadLimit() + + hosts, haveHost := req.Header["Host"] + isH2Upgrade := req.isH2Upgrade() + if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade && req.Method != "CONNECT" { + return nil, http_badRequestError("missing required Host header") + } + if len(hosts) == 1 && !httpguts.ValidHostHeader(hosts[0]) { + return nil, http_badRequestError("malformed Host header") + } + for k, vv := range req.Header { + if !httpguts.ValidHeaderFieldName(k) { + return nil, http_badRequestError("invalid header name") + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + return nil, http_badRequestError("invalid header value") + } + } + } + delete(req.Header, "Host") + + ctx, cancelCtx := context.WithCancel(ctx) + req.ctx = ctx req.RemoteAddr = c.remoteAddr req.TLS = c.tlsState - if body, ok := req.Body.(*body); ok { - body.doEarlyClose = true + var reqBody *http_body + switch b := req.Body.(type) { + case http_noBody: + case *http_body: + reqBody = b + reqBody.doEarlyClose = true + default: + panic(fmt.Errorf("http: unexpected request body type %T", req.Body)) } - w = &response{ + c.rwc.SetReadDeadline(wholeReqDeadline) + + w = &http_response{ conn: c, + cancelCtx: cancelCtx, req: req, - handlerHeader: make(Header), + reqBody: reqBody, + handlerHeader: make(http_Header), contentLength: -1, + + // We populate these ahead of time so we're not + // reading from req.Header after their Handler starts + // and maybe mutates it (Issue 14940) + wants10KeepAlive: req.wantsHttp10KeepAlive(), + wantsClose: req.wantsClose(), + } + if isH2Upgrade { + w.closeAfterReply = true } w.cw.res = w - w.w = newBufioWriterSize(&w.cw, bufferBeforeChunkingSize) + w.w = http_newBufioWriterSize(&w.cw, http_bufferBeforeChunkingSize) return w, nil } -func (w *response) Header() Header { - if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { +// http1ServerSupportsRequest reports whether Go's HTTP/1.x server +// supports the given request. +func http_http1ServerSupportsRequest(req *http_Request) bool { + if req.ProtoMajor == 1 { + return true + } + // Accept "PRI * HTTP/2.0" upgrade requests, so Handlers can + // wire up their own HTTP/2 upgrades. + if req.ProtoMajor == 2 && req.ProtoMinor == 0 && + req.Method == "PRI" && req.RequestURI == "*" { + return true + } + // Reject HTTP/0.x, and all other HTTP/2+ requests (which + // aren't encoded in ASCII anyway). + return false +} - w.cw.header = w.handlerHeader.clone() +func (w *http_response) Header() http_Header { + if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { + // Accessing the header between logically writing it + // and physically writing it means we need to allocate + // a clone to snapshot the logically written state. + w.cw.header = w.handlerHeader.Clone() } w.calledHeader = true return w.handlerHeader @@ -3819,29 +9223,92 @@ // maxPostHandlerReadBytes is the max number of Request.Body bytes not // consumed by a handler that the server will read from the client -// in order to keep a connection alive. If there are more bytes than -// this then the server to be paranoid instead sends a "Connection: -// close" response. +// in order to keep a connection alive. If there are more bytes +// than this, the server, to be paranoid, instead sends a +// "Connection close" response. // // This number is approximately what a typical machine's TCP buffer // size is anyway. (if we have the bytes on the machine, we might as // well read them) -const maxPostHandlerReadBytes = 256 << 10 +const http_maxPostHandlerReadBytes = 256 << 10 -func (w *response) WriteHeader(code int) { +func http_checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at https://httpwg.org/specs/rfc7231.html#status.codes). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + +// relevantCaller searches the call stack for the first function outside of net/http. +// The purpose of this function is to provide more helpful error messages. +func http_relevantCaller() runtime.Frame { + pc := make([]uintptr, 16) + n := runtime.Callers(1, pc) + frames := runtime.CallersFrames(pc[:n]) + var frame runtime.Frame + for { + var more bool + frame, more = frames.Next() + if !strings.HasPrefix(frame.Function, "net/http.") { + return frame + } + if !more { + break + } + } + return frame +} + +func (w *http_response) WriteHeader(code int) { if w.conn.hijacked() { - w.conn.server.logf("http: response.WriteHeader on hijacked connection") + caller := http_relevantCaller() + w.conn.server.logf("http: response.WriteHeader on hijacked connection from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) return } if w.wroteHeader { - w.conn.server.logf("http: multiple response.WriteHeader calls") + caller := http_relevantCaller() + w.conn.server.logf("http: superfluous response.WriteHeader call from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) return } + http_checkWriteHeaderCode(code) + + // Sending a 100 Continue or any non-1XX header disables the + // automatically-sent 100 Continue from Request.Body.Read. If it is a final + // response (200 or higher), we skip draining the request body, which the + // client will never send. + if code == 100 || code >= 200 { + w.disableWriteContinue(code >= 200) + } + + // Handle informational headers. + // + // We shouldn't send any further headers after 101 Switching Protocols, + // so it takes the non-informational path. + if code >= 100 && code <= 199 && code != http_StatusSwitchingProtocols { + http_writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) + + // Per RFC 8297 we must not clear the current header map + w.handlerHeader.WriteSubset(w.conn.bufw, http_excludedHeadersNoBody) + w.conn.bufw.Write(http_crlf) + w.conn.bufw.Flush() + + return + } + w.wroteHeader = true w.status = code if w.calledHeader && w.cw.header == nil { - w.cw.header = w.handlerHeader.clone() + w.cw.header = w.handlerHeader.Clone() } if cl := w.handlerHeader.get("Content-Length"); cl != "" { @@ -3858,7 +9325,7 @@ // extraHeader is the set of headers sometimes added by chunkWriter.writeHeader. // This type is used to avoid extra allocations from cloning and/or populating // the response Header map and all its 1-element slices. -type extraHeader struct { +type http_extraHeader struct { contentType string connection string transferEncoding string @@ -3867,15 +9334,15 @@ } // Sorted the same as extraHeader.Write's loop. -var extraHeaderKeys = [][]byte{ +var http_extraHeaderKeys = [][]byte{ []byte("Content-Type"), []byte("Connection"), []byte("Transfer-Encoding"), } var ( - headerContentLength = []byte("Content-Length: ") - headerDate = []byte("Date: ") + http_headerContentLength = []byte("Content-Length: ") + http_headerDate = []byte("Date: ") ) // Write writes the headers described in h to w. @@ -3883,36 +9350,36 @@ // This method has a value receiver, despite the somewhat large size // of h, because it prevents an allocation. The escape analysis isn't // smart enough to realize this function doesn't mutate h. -func (h extraHeader) Write(w *bufio.Writer) { +func (h http_extraHeader) Write(w *bufio.Writer) { if h.date != nil { - w.Write(headerDate) + w.Write(http_headerDate) w.Write(h.date) - w.Write(crlf) + w.Write(http_crlf) } if h.contentLength != nil { - w.Write(headerContentLength) + w.Write(http_headerContentLength) w.Write(h.contentLength) - w.Write(crlf) + w.Write(http_crlf) } for i, v := range []string{h.contentType, h.connection, h.transferEncoding} { if v != "" { - w.Write(extraHeaderKeys[i]) - w.Write(colonSpace) + w.Write(http_extraHeaderKeys[i]) + w.Write(http_colonSpace) w.WriteString(v) - w.Write(crlf) + w.Write(http_crlf) } } } // writeHeader finalizes the header sent to the client and writes it -// to cw.res.conn.buf. +// to cw.res.conn.bufw. // // p is not written by writeHeader, but is the first chunk of the body -// that will be written. It is sniffed for a Content-Type if none is -// set explicitly. It's also used to set the Content-Length, if the +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. It's also used to set the Content-Length, if the // total body size was small and the handler has already finished // running. -func (cw *chunkWriter) writeHeader(p []byte) { +func (cw *http_chunkWriter) writeHeader(p []byte) { if cw.wroteHeader { return } @@ -3922,6 +9389,11 @@ keepAlivesEnabled := w.conn.server.doKeepAlives() isHEAD := w.req.Method == "HEAD" + // header is written out to w.conn.buf below. Depending on the + // state of the handler, we either own the map or not. If we + // don't own it, the exclude map is created lazily for + // WriteSubset to remove headers. The setHeader struct holds + // headers we need to add. header := cw.header owned := header != nil if !owned { @@ -3941,37 +9413,64 @@ } excludeHeader[key] = true } - var setHeader extraHeader + var setHeader http_extraHeader + // Don't write out the fake "Trailer:foo" keys. See TrailerPrefix. trailers := false + for k := range cw.header { + if strings.HasPrefix(k, http_TrailerPrefix) { + if excludeHeader == nil { + excludeHeader = make(map[string]bool) + } + excludeHeader[k] = true + trailers = true + } + } for _, v := range cw.header["Trailer"] { trailers = true - foreachHeaderElement(v, cw.res.declareTrailer) + http_foreachHeaderElement(v, cw.res.declareTrailer) } te := header.get("Transfer-Encoding") hasTE := te != "" - if w.handlerDone && !trailers && !hasTE && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { + // If the handler is done but never sent a Content-Length + // response header and this is our first (and last) write, set + // it, even to zero. This helps HTTP/1.0 clients keep their + // "keep-alive" connections alive. + // Exceptions: 304/204/1xx responses never get Content-Length, and if + // it was a HEAD request, we don't know the difference between + // 0 actual bytes and 0 bytes because the handler noticed it + // was a HEAD request and chose not to write anything. So for + // HEAD, the handler should either write the Content-Length or + // write non-zero bytes. If it's actually 0 bytes and the + // handler never looked at the Request.Method, we just don't + // send a Content-Length header. + // Further, we don't send an automatic Content-Length if they + // set a Transfer-Encoding, because they're generally incompatible. + if w.handlerDone.Load() && !trailers && !hasTE && http_bodyAllowedForStatus(w.status) && !header.has("Content-Length") && (!isHEAD || len(p) > 0) { w.contentLength = int64(len(p)) setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) } - if w.req.wantsHttp10KeepAlive() && keepAlivesEnabled { + // If this was an HTTP/1.0 request with keep-alive and we sent a + // Content-Length back, we can make this a keep-alive response ... + if w.wants10KeepAlive && keepAlivesEnabled { sentLength := header.get("Content-Length") != "" if sentLength && header.get("Connection") == "keep-alive" { w.closeAfterReply = false } } + // Check for an explicit (and valid) Content-Length header. hasCL := w.contentLength != -1 - if w.req.wantsHttp10KeepAlive() && (isHEAD || hasCL) { + if w.wants10KeepAlive && (isHEAD || hasCL || !http_bodyAllowedForStatus(w.status)) { _, connectionHeaderSet := header["Connection"] if !connectionHeaderSet { setHeader.connection = "keep-alive" } - } else if !w.req.ProtoAtLeast(1, 1) || w.req.wantsClose() { + } else if !w.req.ProtoAtLeast(1, 1) || w.wantsClose { w.closeAfterReply = true } @@ -3979,56 +9478,63 @@ w.closeAfterReply = true } - if ecr, ok := w.req.Body.(*expectContinueReader); ok && !ecr.sawEOF { + // If the client wanted a 100-continue but we never sent it to + // them (or, more strictly: we never finished reading their + // request body), don't reuse this connection. + // + // This behavior was first added on the theory that we don't know + // if the next bytes on the wire are going to be the remainder of + // the request body or the subsequent request (see issue 11549), + // but that's not correct: If we keep using the connection, + // the client is required to send the request body whether we + // asked for it or not. + // + // We probably do want to skip reusing the connection in most cases, + // however. If the client is offering a large request body that we + // don't intend to use, then it's better to close the connection + // than to read the body. For now, assume that if we're sending + // headers, the handler is done reading the body and we should + // drop the connection if we haven't seen EOF. + if w.ecReader != nil && w.reqBody.bodyRemains() { w.closeAfterReply = true } - if w.req.ContentLength != 0 && !w.closeAfterReply { + // We do this by default because there are a number of clients that + // send a full request before starting to read the response, and they + // can deadlock if we start writing the response with unconsumed body + // remaining. See Issue 15527 for some history. + // + // If full duplex mode has been enabled with ResponseController.EnableFullDuplex, + // then leave the request body alone. + // + // We don't take this path when w.closeAfterReply is set. + // We may not need to consume the request to get ready for the next one + // (since we're closing the conn), but a client which sends a full request + // before reading a response may deadlock in this case. + // This behavior has been present since CL 5268043 (2011), however, + // so it doesn't seem to be causing problems. + if w.req.ContentLength != 0 && w.reqBody != nil && !w.closeAfterReply && !w.fullDuplex { var discard, tooBig bool - - switch bdy := w.req.Body.(type) { - case *expectContinueReader: - if bdy.resp.wroteContinue { - discard = true + w.reqBody.mu.Lock() + switch { + case w.reqBody.closed: + if !w.reqBody.sawEOF { + // Body was closed in handler with non-EOF error. + w.closeAfterReply = true } - case *body: - bdy.mu.Lock() - switch { - case bdy.closed: - if !bdy.sawEOF { - - w.closeAfterReply = true - } - case bdy.unreadDataSizeLocked() >= maxPostHandlerReadBytes: - tooBig = true - default: - discard = true - } - bdy.mu.Unlock() + case w.reqBody.unreadDataSizeLocked() >= http_maxPostHandlerReadBytes: + tooBig = true default: discard = true } + w.reqBody.mu.Unlock() if discard { - _, err := io.CopyN(io.Discard, w.req.Body, maxPostHandlerReadBytes+1) - switch err { - case nil: - - tooBig = true - case ErrBodyReadAfterClose: - - case io.EOF: - - err = w.req.Body.Close() - if err != nil { - w.closeAfterReply = true - } - default: - + w.reqBody.Close() + if w.reqBody.didEarlyClose() { w.closeAfterReply = true } } - if tooBig { w.requestTooLarge() delHeader("Connection") @@ -4037,52 +9543,71 @@ } code := w.status - if bodyAllowedForStatus(code) { - + if http_bodyAllowedForStatus(code) { + // If no content type, apply sniffing algorithm to body. _, haveType := header["Content-Type"] - if !haveType && !hasTE { - setHeader.contentType = DetectContentType(p) + + // If the Content-Encoding was set and is non-blank, + // we shouldn't sniff the body. See Issue 31753. + ce := header.Get("Content-Encoding") + hasCE := len(ce) > 0 + if !hasCE && !haveType && !hasTE && len(p) > 0 { + setHeader.contentType = http_DetectContentType(p) } } else { - for _, k := range suppressedHeaders(code) { + for _, k := range http_suppressedHeaders(code) { delHeader(k) } } - if _, ok := header["Date"]; !ok { - setHeader.date = appendTime(cw.res.dateBuf[:0], time.Now()) + if !header.has("Date") { + setHeader.date = time.Now().UTC().AppendFormat(cw.res.dateBuf[:0], http_TimeFormat) } if hasCL && hasTE && te != "identity" { - + // TODO: return an error if WriteHeader gets a return parameter + // For now just ignore the Content-Length. w.conn.server.logf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d", te, w.contentLength) delHeader("Content-Length") hasCL = false } - if w.req.Method == "HEAD" || !bodyAllowedForStatus(code) { - - } else if code == StatusNoContent { + if w.req.Method == "HEAD" || !http_bodyAllowedForStatus(code) || code == http_StatusNoContent { + // Response has no body. delHeader("Transfer-Encoding") } else if hasCL { + // Content-Length has been provided, so no chunking is to be done. delHeader("Transfer-Encoding") } else if w.req.ProtoAtLeast(1, 1) { - + // HTTP/1.1 or greater: Transfer-Encoding has been set to identity, and no + // content-length has been provided. The connection must be closed after the + // reply is written, and no chunking is to be done. This is the setup + // recommended in the Server-Sent Events candidate recommendation 11, + // section 8. if hasTE && te == "identity" { cw.chunking = false w.closeAfterReply = true + delHeader("Transfer-Encoding") } else { - + // HTTP/1.1 or greater: use chunked transfer encoding + // to avoid closing the connection at EOF. cw.chunking = true setHeader.transferEncoding = "chunked" + if hasTE && te == "chunked" { + // We will send the chunked Transfer-Encoding header later. + delHeader("Transfer-Encoding") + } } } else { - + // HTTP version < 1.1: cannot do chunked transfer + // encoding and we don't know the Content-Length so + // signal EOF by closing connection. w.closeAfterReply = true - delHeader("Transfer-Encoding") + delHeader("Transfer-Encoding") // in case already set } + // Cannot use Content-Length with non-identity Transfer-Encoding. if cw.chunking { delHeader("Content-Length") } @@ -4090,22 +9615,28 @@ return } - if w.closeAfterReply && (!keepAlivesEnabled || !hasToken(cw.header.get("Connection"), "close")) { + // Only override the Connection header if it is not a successful + // protocol switch response and if KeepAlives are not enabled. + // See https://golang.org/issue/36381. + delConnectionHeader := w.closeAfterReply && + (!keepAlivesEnabled || !http_hasToken(cw.header.get("Connection"), "close")) && + !http_isProtocolSwitchResponse(w.status, header) + if delConnectionHeader { delHeader("Connection") if w.req.ProtoAtLeast(1, 1) { setHeader.connection = "close" } } - w.conn.buf.WriteString(statusLine(w.req, code)) - cw.header.WriteSubset(w.conn.buf, excludeHeader) - setHeader.Write(w.conn.buf.Writer) - w.conn.buf.Write(crlf) + http_writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) + cw.header.WriteSubset(w.conn.bufw, excludeHeader) + setHeader.Write(w.conn.bufw) + w.conn.bufw.Write(http_crlf) } // foreachHeaderElement splits v according to the "#rule" construction -// in RFC 2616 section 2.1 and calls fn for each non-empty element. -func foreachHeaderElement(v string, fn func(string)) { +// in RFC 7230 section 7 and calls fn for each non-empty element. +func http_foreachHeaderElement(v string, fn func(string)) { v = textproto.TrimString(v) if v == "" { return @@ -4114,70 +9645,47 @@ fn(v) return } - for _, f := range strings.Split(v, ",") { + for f := range strings.SplitSeq(v, ",") { if f = textproto.TrimString(f); f != "" { fn(f) } } } -// statusLines is a cache of Status-Line strings, keyed by code (for -// HTTP/1.1) or negative code (for HTTP/1.0). This is faster than a -// map keyed by struct of two fields. This map's max size is bounded -// by 2*len(statusText), two protocol types for each known official -// status code in the statusText map. -var ( - statusMu sync.RWMutex - statusLines = make(map[int]string) -) - -// statusLine returns a response Status-Line (RFC 2616 Section 6.1) -// for the given request and response status code. -func statusLine(req *Request, code int) string { - - key := code - proto11 := req.ProtoAtLeast(1, 1) - if !proto11 { - key = -key +// writeStatusLine writes an HTTP/1.x Status-Line (RFC 7230 Section 3.1.2) +// to bw. is11 is whether the HTTP request is HTTP/1.1. false means HTTP/1.0. +// code is the response status code. +// scratch is an optional scratch buffer. If it has at least capacity 3, it's used. +func http_writeStatusLine(bw *bufio.Writer, is11 bool, code int, scratch []byte) { + if is11 { + bw.WriteString("HTTP/1.1 ") + } else { + bw.WriteString("HTTP/1.0 ") } - statusMu.RLock() - line, ok := statusLines[key] - statusMu.RUnlock() - if ok { - return line + if text := http_StatusText(code); text != "" { + bw.Write(strconv.AppendInt(scratch[:0], int64(code), 10)) + bw.WriteByte(' ') + bw.WriteString(text) + bw.WriteString("\r\n") + } else { + // don't worry about performance + fmt.Fprintf(bw, "%03d status code %d\r\n", code, code) } - - proto := "HTTP/1.0" - if proto11 { - proto = "HTTP/1.1" - } - codestring := strconv.Itoa(code) - text, ok := statusText[code] - if !ok { - text = "status code " + codestring - } - line = proto + " " + codestring + " " + text + "\r\n" - if ok { - statusMu.Lock() - defer statusMu.Unlock() - statusLines[key] = line - } - return line } // bodyAllowed reports whether a Write is allowed for this response type. // It's illegal to call this before the header has been flushed. -func (w *response) bodyAllowed() bool { +func (w *http_response) bodyAllowed() bool { if !w.wroteHeader { - panic("") + panic("net/http: bodyAllowed called before the header was written") } - return bodyAllowedForStatus(w.status) + return http_bodyAllowedForStatus(w.status) } // The Life Of A Write is like this: // // Handler starts. No header has been sent. The handler can either -// write a header, or just start writing. Writing before sending a header +// write a header, or just start writing. Writing before sending a header // sends an implicitly empty 200 OK header. // // If the handler didn't declare a Content-Length up front, we either @@ -4190,51 +9698,60 @@ // // The Writers are wired together like: // -// 1. *response (the ResponseWriter) -> -// 2. (*response).w, a *bufio.Writer of bufferBeforeChunkingSize bytes -// 3. chunkWriter.Writer (whose writeHeader finalizes Content-Length/Type) -// and which writes the chunk headers, if needed. -// 4. conn.buf, a bufio.Writer of default (4kB) bytes, writing to -> -// 5. checkConnErrorWriter{c}, which notes any non-nil error on Write -// and populates c.werr with it if so. but otherwise writes to: -// 6. the rwc, the net.Conn. +// 1. *response (the ResponseWriter) -> +// 2. (*response).w, a [*bufio.Writer] of bufferBeforeChunkingSize bytes -> +// 3. chunkWriter.Writer (whose writeHeader finalizes Content-Length/Type) +// and which writes the chunk headers, if needed -> +// 4. conn.bufw, a *bufio.Writer of default (4kB) bytes, writing to -> +// 5. checkConnErrorWriter{c}, which notes any non-nil error on Write +// and populates c.werr with it if so, but otherwise writes to -> +// 6. the rwc, the [net.Conn]. // // TODO(bradfitz): short-circuit some of the buffering when the // initial header contains both a Content-Type and Content-Length. // Also short-circuit in (1) when the header's been sent and not in // chunking mode, writing directly to (4) instead, if (2) has no -// buffered data. More generally, we could short-circuit from (1) to +// buffered data. More generally, we could short-circuit from (1) to // (3) even in chunking mode if the write size from (1) is over some // threshold and nothing is in (2). The answer might be mostly making // bufferBeforeChunkingSize smaller and having bufio's fast-paths deal // with this instead. -func (w *response) Write(data []byte) (n int, err error) { +func (w *http_response) Write(data []byte) (n int, err error) { return w.write(len(data), data, "") } -func (w *response) WriteString(data string) (n int, err error) { +func (w *http_response) WriteString(data string) (n int, err error) { return w.write(len(data), nil, data) } // either dataB or dataS is non-zero. -func (w *response) write(lenData int, dataB []byte, dataS string) (n int, err error) { +func (w *http_response) write(lenData int, dataB []byte, dataS string) (n int, err error) { if w.conn.hijacked() { - w.conn.server.logf("http: response.Write on hijacked connection") - return 0, ErrHijacked + if lenData > 0 { + caller := http_relevantCaller() + w.conn.server.logf("http: response.Write on hijacked connection from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + } + return 0, http_ErrHijacked } + + if w.canWriteContinue.Load() { + // Body reader wants to write 100 Continue but hasn't yet. Tell it not to. + w.disableWriteContinue(true) + } + if !w.wroteHeader { - w.WriteHeader(StatusOK) + w.WriteHeader(http_StatusOK) } if lenData == 0 { return 0, nil } if !w.bodyAllowed() { - return 0, ErrBodyNotAllowed + return 0, http_ErrBodyNotAllowed } - w.written += int64(lenData) + w.written += int64(lenData) // ignoring errors, for errorKludge if w.contentLength != -1 && w.written > w.contentLength { - return 0, ErrContentLength + return 0, http_ErrContentLength } if dataB != nil { return w.w.Write(dataB) @@ -4243,19 +9760,30 @@ } } -func (w *response) finishRequest() { - w.handlerDone = true +func (w *http_response) finishRequest() { + w.handlerDone.Store(true) if !w.wroteHeader { - w.WriteHeader(StatusOK) + w.WriteHeader(http_StatusOK) } w.w.Flush() - putBufioWriter(w.w) + http_putBufioWriter(w.w) w.cw.close() - w.conn.buf.Flush() + w.conn.bufw.Flush() - w.req.Body.Close() + w.conn.r.abortPendingRead() + w.reqBody.registerOnHitEOF(nil) // prevent new background read from starting + + if w.canWriteContinue.Load() { + w.disableWriteContinue(true) + } + + // Close the body (regardless of w.closeAfterReply) so we can + // re-use its bufio.Reader later safely. + // + // In full-duplex mode, this may also drain the remaining request body. + w.reqBody.Close() if w.req.MultipartForm != nil { w.req.MultipartForm.RemoveAll() @@ -4264,17 +9792,21 @@ // shouldReuseConnection reports whether the underlying TCP connection can be reused. // It must only be called after the handler is done executing. -func (w *response) shouldReuseConnection() bool { +func (w *http_response) shouldReuseConnection() bool { if w.closeAfterReply { - + // The request or something set while executing the + // handler indicated we shouldn't reuse this + // connection. return false } if w.req.Method != "HEAD" && w.contentLength != -1 && w.bodyAllowed() && w.contentLength != w.written { - + // Did not write enough. Avoid getting out of sync. return false } + // There was some error writing to the underlying connection + // during the request, so don't re-use this conn. if w.conn.werr != nil { return false } @@ -4286,38 +9818,47 @@ return true } -func (w *response) closedRequestBodyEarly() bool { - body, ok := w.req.Body.(*body) - return ok && body.didEarlyClose() +func (w *http_response) closedRequestBodyEarly() bool { + return w.reqBody != nil && w.reqBody.didEarlyClose() } -func (w *response) Flush() { +func (w *http_response) Flush() { + w.FlushError() +} + +func (w *http_response) FlushError() error { if !w.wroteHeader { - w.WriteHeader(StatusOK) + w.WriteHeader(http_StatusOK) } - w.w.Flush() - w.cw.flush() + err := w.w.Flush() + e2 := w.cw.flush() + if err == nil { + err = e2 + } + return err } -func (c *conn) finalFlush() { - if c.buf != nil { - c.buf.Flush() +func (c *http_conn) finalFlush() { + if c.bufr != nil { + // Steal the bufio.Reader (~4KB worth of memory) and its associated + // reader for a future connection. + http_putBufioReader(c.bufr) + c.bufr = nil + } - putBufioReader(c.buf.Reader) - - putBufioWriter(c.buf.Writer) - - c.buf = nil + if c.bufw != nil { + c.bufw.Flush() + // Steal the bufio.Writer (~4KB worth of memory) and its associated + // writer for a future connection. + http_putBufioWriter(c.bufw) + c.bufw = nil } } // Close the connection. -func (c *conn) close() { +func (c *http_conn) close() { c.finalFlush() - if c.rwc != nil { - c.rwc.Close() - c.rwc = nil - } + c.rwc.Close() } // rstAvoidanceDelay is the amount of time we sleep after closing the @@ -4326,34 +9867,58 @@ // and processes its final data before they process the subsequent RST // from closing a connection with known unread data. // This RST seems to occur mostly on BSD systems. (And Windows?) -// This timeout is somewhat arbitrary (~latency around the planet). -const rstAvoidanceDelay = 500 * time.Millisecond +// This timeout is somewhat arbitrary (~latency around the planet), +// and may be modified by tests. +// +// TODO(bcmills): This should arguably be a server configuration parameter, +// not a hard-coded value. +var http_rstAvoidanceDelay = 500 * time.Millisecond -type closeWriter interface { +type http_closeWriter interface { CloseWrite() error } -var _ closeWriter = (*net.TCPConn)(nil) +var _ http_closeWriter = (*net.TCPConn)(nil) -// closeWrite flushes any outstanding data and sends a FIN packet (if -// client is connected via TCP), signalling that we're done. We then +// closeWriteAndWait flushes any outstanding data and sends a FIN packet (if +// client is connected via TCP), signaling that we're done. We then // pause for a bit, hoping the client processes it before any // subsequent RST. // // See https://golang.org/issue/3595 -func (c *conn) closeWriteAndWait() { +func (c *http_conn) closeWriteAndWait() { c.finalFlush() - if tcp, ok := c.rwc.(closeWriter); ok { + if tcp, ok := c.rwc.(http_closeWriter); ok { tcp.CloseWrite() } - time.Sleep(rstAvoidanceDelay) + + // When we return from closeWriteAndWait, the caller will fully close the + // connection. If client is still writing to the connection, this will cause + // the write to fail with ECONNRESET or similar. Unfortunately, many TCP + // implementations will also drop unread packets from the client's read buffer + // when a write fails, causing our final response to be truncated away too. + // + // As a result, https://www.rfc-editor.org/rfc/rfc7230#section-6.6 recommends + // that “[t]he server … continues to read from the connection until it + // receives a corresponding close by the client, or until the server is + // reasonably certain that its own TCP stack has received the client's + // acknowledgement of the packet(s) containing the server's last response.” + // + // Unfortunately, we have no straightforward way to be “reasonably certain” + // that we have received the client's ACK, and at any rate we don't want to + // allow a misbehaving client to soak up server connections indefinitely by + // withholding an ACK, nor do we want to go through the complexity or overhead + // of using low-level APIs to figure out when a TCP round-trip has completed. + // + // Instead, we declare that we are “reasonably certain” that we received the + // ACK if maxRSTAvoidanceDelay has elapsed. + time.Sleep(http_rstAvoidanceDelay) } -// validNPN reports whether the proto is not a blacklisted Next -// Protocol Negotiation protocol. Empty and built-in protocol types -// are blacklisted and can't be overridden with alternate -// implementations. -func validNPN(proto string) bool { +// validNextProto reports whether the proto is a valid ALPN protocol name. +// Everything is valid except the empty string and built-in protocol types, +// so that those can't be overridden with alternate implementations. +func http_validNextProto(proto string) bool { switch proto { case "", "http/1.1", "http/1.0": return false @@ -4361,239 +9926,652 @@ return true } -func (c *conn) setState(nc net.Conn, state ConnState) { - if hook := c.server.ConnState; hook != nil { +const ( + http_runHooks = true + http_skipHooks = false +) + +func (c *http_conn) setState(nc net.Conn, state http_ConnState, runHook bool) { + srv := c.server + switch state { + case http_StateNew: + srv.trackConn(c, true) + case http_StateHijacked, http_StateClosed: + srv.trackConn(c, false) + } + if state > 0xff || state < 0 { + panic("internal error") + } + packedState := uint64(time.Now().Unix()<<8) | uint64(state) + c.curState.Store(packedState) + if !runHook { + return + } + if hook := srv.ConnState; hook != nil { hook(nc, state) } } +func (c *http_conn) getState() (state http_ConnState, unixSec int64) { + packedState := c.curState.Load() + return http_ConnState(packedState & 0xff), int64(packedState >> 8) +} + +// badRequestError is a literal string (used by in the server in HTML, +// unescaped) to tell the user why their request was bad. It should +// be plain text without user info or other embedded errors. +func http_badRequestError(e string) error { return http_statusError{http_StatusBadRequest, e} } + +// statusError is an error used to respond to a request with an HTTP status. +// The text should be plain text without user info or other embedded errors. +type http_statusError struct { + code int + text string +} + +func (e http_statusError) Error() string { return http_StatusText(e.code) + ": " + e.text } + +// ErrAbortHandler is a sentinel panic value to abort a handler. +// While any panic from ServeHTTP aborts the response to the client, +// panicking with ErrAbortHandler also suppresses logging of a stack +// trace to the server's error log. +var http_ErrAbortHandler = internal.ErrAbortHandler + +// isCommonNetReadError reports whether err is a common error +// encountered during reading a request off the network when the +// client has gone away or had its read fail somehow. This is used to +// determine which logs are interesting enough to log about. +func http_isCommonNetReadError(err error) bool { + if err == io.EOF { + return true + } + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + return true + } + if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { + return true + } + return false +} + // Serve a new connection. -func (c *conn) serve() { - origConn := c.rwc +func (c *http_conn) serve(ctx context.Context) { + if ra := c.rwc.RemoteAddr(); ra != nil { + c.remoteAddr = ra.String() + } + ctx = context.WithValue(ctx, http_LocalAddrContextKey, c.rwc.LocalAddr()) + var inFlightResponse *http_response defer func() { - if err := recover(); err != nil { + if err := recover(); err != nil && err != http_ErrAbortHandler { const size = 64 << 10 buf := make([]byte, size) buf = buf[:runtime.Stack(buf, false)] c.server.logf("http: panic serving %v: %v\n%s", c.remoteAddr, err, buf) } + if inFlightResponse != nil { + inFlightResponse.cancelCtx() + inFlightResponse.disableWriteContinue(true) + } if !c.hijacked() { + if inFlightResponse != nil { + inFlightResponse.conn.r.abortPendingRead() + inFlightResponse.reqBody.Close() + } c.close() - c.setState(origConn, StateClosed) + c.setState(c.rwc, http_StateClosed, http_runHooks) } }() - if tlsConn, ok := c.rwc.(*tls.Conn); ok { - if d := c.server.ReadTimeout; d != 0 { - c.rwc.SetReadDeadline(time.Now().Add(d)) + type connectionStater interface { + ConnectionState() tls.ConnectionState + } + type handshakeContexter interface { + HandshakeContext(ctx context.Context) error + } + if connStater, ok := c.rwc.(connectionStater); ok { + tlsTO := c.server.tlsHandshakeTimeout() + if tlsTO > 0 { + dl := time.Now().Add(tlsTO) + c.rwc.SetReadDeadline(dl) + c.rwc.SetWriteDeadline(dl) } - if d := c.server.WriteTimeout; d != 0 { - c.rwc.SetWriteDeadline(time.Now().Add(d)) + var err error + if handshaker, ok := c.rwc.(handshakeContexter); ok { + err = handshaker.HandshakeContext(ctx) } - if err := tlsConn.Handshake(); err != nil { - c.server.logf("http: TLS handshake error from %s: %v", c.rwc.RemoteAddr(), err) + if err != nil { + // If the handshake failed due to the client not speaking + // TLS, assume they're speaking plaintext HTTP and write a + // 400 response on the TLS conn's underlying net.Conn. + var reason string + if re, ok := err.(tls.RecordHeaderError); ok && re.Conn != nil && http_tlsRecordHeaderLooksLikeHTTP(re.RecordHeader) { + io.WriteString(re.Conn, "HTTP/1.0 400 Bad Request\r\n\r\nClient sent an HTTP request to an HTTPS server.\n") + re.Conn.Close() + reason = "client sent an HTTP request to an HTTPS server" + } else { + reason = err.Error() + } + c.server.logf("http: TLS handshake error from %s: %v", c.rwc.RemoteAddr(), reason) return } + // Restore Conn-level deadlines. + if tlsTO > 0 { + c.rwc.SetReadDeadline(time.Time{}) + c.rwc.SetWriteDeadline(time.Time{}) + } c.tlsState = new(tls.ConnectionState) - *c.tlsState = tlsConn.ConnectionState() - if proto := c.tlsState.NegotiatedProtocol; validNPN(proto) { + *c.tlsState = connStater.ConnectionState() + proto := c.tlsState.NegotiatedProtocol + if proto == "h2" && c.server.h2 != nil { + // net/http/internal/http2 path. + // + // Mark freshly created HTTP/2 as active and prevent any server state hooks + // from being run on these connections. This prevents closeIdleConns from + // closing such connections. See issue https://golang.org/issue/39776. + c.setState(c.rwc, http_StateActive, http_skipHooks) + const sawClientPreface = false + c.server.serveHTTP2Conn(ctx, c.rwc, http_serverHandler{c.server}, sawClientPreface, nil, nil) + return + } + tlsConn, tlsConnOK := c.rwc.(*tls.Conn) + if http_validNextProto(proto) && tlsConnOK { + // Legacy TLSNextProto path. if fn := c.server.TLSNextProto[proto]; fn != nil { - h := initNPNRequest{tlsConn, serverHandler{c.server}} + h := http_initALPNRequest{ctx, tlsConn, http_serverHandler{c.server}} + // Mark freshly created HTTP/2 as active (see above). + c.setState(c.rwc, http_StateActive, http_skipHooks) fn(c.server, tlsConn, h) } return } } - for { - w, err := c.readRequest() - if c.lr.N != c.server.initialLimitedReaderSize() { + // HTTP/1.x or unencrypted HTTP/2. - c.setState(c.rwc, StateActive) + // Set Request.TLS if the conn is not a *tls.Conn, but implements ConnectionState. + if c.tlsState == nil { + if tc, ok := c.rwc.(connectionStater); ok { + c.tlsState = new(tls.ConnectionState) + *c.tlsState = tc.ConnectionState() + } + } + + ctx, cancelCtx := context.WithCancel(ctx) + c.cancelCtx = cancelCtx + defer cancelCtx() + + c.r = &http_connReader{conn: c, rwc: c.rwc} + c.bufr = http_newBufioReader(c.r) + c.bufw = http_newBufioWriterSize(http_checkConnErrorWriter{c}, 4<<10) + + if d := c.server.readHeaderTimeout(); d > 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + } + + protos := c.server.protocols() + if c.tlsState == nil && protos.UnencryptedHTTP2() { + if c.maybeServeUnencryptedHTTP2(ctx) { + return + } + } + if !protos.HTTP1() { + return + } + + // HTTP/1.x from here on. + + for { + w, err := c.readRequest(ctx) + if c.r.remain != c.server.initialReadLimitSize() { + // If we read any bytes off the wire, we're active. + c.setState(c.rwc, http_StateActive, http_runHooks) + } + if c.server.shuttingDown() { + return } if err != nil { - if err == errTooLarge { + const errorHeaders = "\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n" - io.WriteString(c.rwc, "HTTP/1.1 413 Request Entity Too Large\r\n\r\n") + switch { + case err == http_errTooLarge: + // Their HTTP client may or may not be + // able to read this if we're + // responding to them and hanging up + // while they're still writing their + // request. Undefined behavior. + const publicErr = "431 Request Header Fields Too Large" + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) c.closeWriteAndWait() - break - } else if err == io.EOF { - break - } else if neterr, ok := err.(net.Error); ok && neterr.Timeout() { - break + return + + case http_isUnsupportedTEError(err): + // Respond as per RFC 7230 Section 3.3.1 which says, + // A server that receives a request message with a + // transfer coding it does not understand SHOULD + // respond with 501 (Unimplemented). + code := http_StatusNotImplemented + + // We purposefully aren't echoing back the transfer-encoding's value, + // so as to mitigate the risk of cross side scripting by an attacker. + fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s%sUnsupported transfer encoding", code, http_StatusText(code), errorHeaders) + return + + case http_isCommonNetReadError(err): + return // don't reply + + default: + if v, ok := err.(http_statusError); ok { + fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s: %s%s%d %s: %s", v.code, http_StatusText(v.code), v.text, errorHeaders, v.code, http_StatusText(v.code), v.text) + return + } + const publicErr = "400 Bad Request" + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) + return } - io.WriteString(c.rwc, "HTTP/1.1 400 Bad Request\r\n\r\n") - break } + // Expect 100 Continue support req := w.req if req.expectsContinue() { if req.ProtoAtLeast(1, 1) && req.ContentLength != 0 { - - req.Body = &expectContinueReader{readCloser: req.Body, resp: w} + // Wrap the Body reader with one that replies on the connection + w.ecReader = &http_expectContinueReader{readCloser: req.Body, resp: w} + w.canWriteContinue.Store(true) + req.Body = w.ecReader } - req.Header.Del("Expect") } else if req.Header.get("Expect") != "" { w.sendExpectationFailed() - break + return } - serverHandler{c.server}.ServeHTTP(w, w.req) + c.curReq.Store(w) + + // Start background read, which detects when a client has closed its connection + // while a request handler is still running. When the request has a body, we + // start the background read only after the entire body has been consumed. + if w.reqBody.bodyRemains() { + w.reqBody.registerOnHitEOF(w.conn.r.startBackgroundRead) + } else { + w.conn.r.startBackgroundRead() + } + + // HTTP cannot have multiple simultaneous active requests.[*] + // Until the server replies to this request, it can't read another, + // so we might as well run the handler in this goroutine. + // [*] Not strictly true: HTTP pipelining. We could let them all process + // in parallel even if their responses need to be serialized. + // But we're not going to implement HTTP pipelining because it + // was never deployed in the wild and the answer is HTTP/2. + inFlightResponse = w + http_serverHandler{c.server}.ServeHTTP(w, w.req) + inFlightResponse = nil + w.cancelCtx() if c.hijacked() { + c.r.releaseConn() return } w.finishRequest() + c.rwc.SetWriteDeadline(time.Time{}) if !w.shouldReuseConnection() { if w.requestBodyLimitHit || w.closedRequestBodyEarly() { c.closeWriteAndWait() } - break + return } - c.setState(c.rwc, StateIdle) + c.setState(c.rwc, http_StateIdle, http_runHooks) + c.curReq.Store(nil) + + if !w.conn.server.doKeepAlives() { + // We're in shutdown mode. We might've replied + // to the user without "Connection: close" and + // they might think they can send another + // request, but such is life with HTTP/1.1. + return + } + + if d := c.server.idleTimeout(); d > 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + } else { + c.rwc.SetReadDeadline(time.Time{}) + } + + // Wait for the connection to become readable again before trying to + // read the next request. This prevents a ReadHeaderTimeout or + // ReadTimeout from starting until the first bytes of the next request + // have been received. + if _, err := c.bufr.Peek(4); err != nil { + return + } + + if d := c.server.readHeaderTimeout(); d > 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + } else { + c.rwc.SetReadDeadline(time.Time{}) + } } } -func (w *response) sendExpectationFailed() { +// unencryptedHTTP2Request is an HTTP handler that initializes +// certain uninitialized fields in its *Request. +// +// It's the unencrypted version of initALPNRequest. +type http_unencryptedHTTP2Request struct { + ctx context.Context + c net.Conn + h http_serverHandler +} +func (h http_unencryptedHTTP2Request) BaseContext() context.Context { return h.ctx } + +func (h http_unencryptedHTTP2Request) ServeHTTP(rw http_ResponseWriter, req *http_Request) { + if req.Body == nil { + req.Body = http_NoBody + } + if req.RemoteAddr == "" { + req.RemoteAddr = h.c.RemoteAddr().String() + } + h.h.ServeHTTP(rw, req) +} + +// unencryptedNetConnInTLSConn is used to pass an unencrypted net.Conn to +// functions that only accept a *tls.Conn. +type http_unencryptedNetConnInTLSConn struct { + net.Conn // panic on all net.Conn methods + conn net.Conn +} + +func (c http_unencryptedNetConnInTLSConn) UnencryptedNetConn() net.Conn { + return c.conn +} + +func http_unencryptedTLSConn(c net.Conn) *tls.Conn { + return tls.Client(http_unencryptedNetConnInTLSConn{conn: c}, nil) +} + +// TLSNextProto key to use for unencrypted HTTP/2 connections. +// Not actually a TLS-negotiated protocol. +const http_nextProtoUnencryptedHTTP2 = "unencrypted_http2" + +func (c *http_conn) maybeServeUnencryptedHTTP2(ctx context.Context) bool { + var nextFunc func(*http_Server, *tls.Conn, http_Handler) + if c.server.h2 == nil { + var ok bool + nextFunc, ok = c.server.TLSNextProto[http_nextProtoUnencryptedHTTP2] + if !ok { + return false + } + } + hasPreface := func(c *http_conn, preface []byte) bool { + c.r.setReadLimit(int64(len(preface)) - int64(c.bufr.Buffered())) + got, err := c.bufr.Peek(len(preface)) + c.r.setInfiniteReadLimit() + return err == nil && bytes.Equal(got, preface) + } + if !hasPreface(c, []byte("PRI * HTTP/2.0")) { + return false + } + if !hasPreface(c, []byte("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")) { + return false + } + c.setState(c.rwc, http_StateActive, http_skipHooks) + if c.server.h2 != nil { + const sawClientPreface = true + c.server.serveHTTP2Conn(ctx, c.rwc, http_serverHandler{c.server}, sawClientPreface, nil, nil) + } else { + h := http_unencryptedHTTP2Request{ctx, c.rwc, http_serverHandler{c.server}} + nextFunc(c.server, http_unencryptedTLSConn(c.rwc), h) + } + return true +} + +func (w *http_response) sendExpectationFailed() { + // TODO(bradfitz): let ServeHTTP handlers handle + // requests with non-standard expectation[s]? Seems + // theoretical at best, and doesn't fit into the + // current ServeHTTP model anyway. We'd need to + // make the ResponseWriter an optional + // "ExpectReplier" interface or something. + // + // For now we'll just obey RFC 7231 5.1.1 which says + // "A server that receives an Expect field-value other + // than 100-continue MAY respond with a 417 (Expectation + // Failed) status code to indicate that the unexpected + // expectation cannot be met." w.Header().Set("Connection", "close") - w.WriteHeader(StatusExpectationFailed) + w.WriteHeader(http_StatusExpectationFailed) w.finishRequest() } -// Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter -// and a Hijacker. -func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { +// Hijack implements the [Hijacker.Hijack] method. Our response is both a [ResponseWriter] +// and a [Hijacker]. +func (w *http_response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { + if w.handlerDone.Load() { + panic("net/http: Hijack called after ServeHTTP finished") + } + w.disableWriteContinue(false) if w.wroteHeader { w.cw.flush() } - rwc, buf, err = w.conn.hijack() + c := w.conn + c.mu.Lock() + defer c.mu.Unlock() + + // Release the bufioWriter that writes to the chunk writer, it is not + // used after a connection has been hijacked. + rwc, buf, err = c.hijackLocked() if err == nil { - putBufioWriter(w.w) + http_putBufioWriter(w.w) w.w = nil } return rwc, buf, err } -func (w *response) CloseNotify() <-chan bool { - return w.conn.closeNotify() +func (w *http_response) CloseNotify() <-chan bool { + w.lazyCloseNotifyMu.Lock() + defer w.lazyCloseNotifyMu.Unlock() + if w.handlerDone.Load() { + panic("net/http: CloseNotify called after ServeHTTP finished") + } + if w.closeNotifyCh == nil { + w.closeNotifyCh = make(chan bool, 1) + if w.closeNotifyTriggered { + w.closeNotifyCh <- true // action prior closeNotify call + } + } + return w.closeNotifyCh +} + +func (w *http_response) closeNotify() { + w.lazyCloseNotifyMu.Lock() + defer w.lazyCloseNotifyMu.Unlock() + if w.closeNotifyTriggered { + return // already triggered + } + w.closeNotifyTriggered = true + if w.closeNotifyCh != nil { + w.closeNotifyCh <- true + } } // The HandlerFunc type is an adapter to allow the use of -// ordinary functions as HTTP handlers. If f is a function +// ordinary functions as HTTP handlers. If f is a function // with the appropriate signature, HandlerFunc(f) is a -// Handler object that calls f. -type HandlerFunc func(ResponseWriter, *Request) +// [Handler] that calls f. +type http_HandlerFunc func(http_ResponseWriter, *http_Request) // ServeHTTP calls f(w, r). -func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { +func (f http_HandlerFunc) ServeHTTP(w http_ResponseWriter, r *http_Request) { f(w, r) } +// Helper handlers + // Error replies to the request with the specified error message and HTTP code. +// It does not otherwise end the request; the caller should ensure no further +// writes are done to w. // The error message should be plain text. -func Error(w ResponseWriter, error string, code int) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Content-Type-Options", "nosniff") +// +// Error deletes the Content-Length header, +// sets Content-Type to “text/plain; charset=utf-8”, +// and sets X-Content-Type-Options to “nosniff”. +// This configures the header properly for the error message, +// in case the caller had set it up expecting a successful output. +func http_Error(w http_ResponseWriter, error string, code int) { + h := w.Header() + + // Delete the Content-Length header, which might be for some other content. + // Assuming the error string fits in the writer's buffer, we'll figure + // out the correct Content-Length for it later. + // + // We don't delete Content-Encoding, because some middleware sets + // Content-Encoding: gzip and wraps the ResponseWriter to compress on-the-fly. + // See https://go.dev/issue/66343. + h.Del("Content-Length") + + // There might be content type already set, but we reset it to + // text/plain for the error message. + h.Set("Content-Type", "text/plain; charset=utf-8") + h.Set("X-Content-Type-Options", "nosniff") w.WriteHeader(code) fmt.Fprintln(w, error) } // NotFound replies to the request with an HTTP 404 not found error. -func NotFound(w ResponseWriter, r *Request) { Error(w, "404 page not found", StatusNotFound) } +func http_NotFound(w http_ResponseWriter, r *http_Request) { + http_Error(w, "404 page not found", http_StatusNotFound) +} // NotFoundHandler returns a simple request handler -// that replies to each request with a ` + "`" + `` + "`" + `404 page not found'' reply. -func NotFoundHandler() Handler { return HandlerFunc(NotFound) } +// that replies to each request with a “404 page not found” reply. +func http_NotFoundHandler() http_Handler { return http_HandlerFunc(http_NotFound) } -// StripPrefix returns a handler that serves HTTP requests -// by removing the given prefix from the request URL's Path -// and invoking the handler h. StripPrefix handles a -// request for a path that doesn't begin with prefix by -// replying with an HTTP 404 not found error. -func StripPrefix(prefix string, h Handler) Handler { +// StripPrefix returns a handler that serves HTTP requests by removing the +// given prefix from the request URL's Path (and RawPath if set) and invoking +// the handler h. StripPrefix handles a request for a path that doesn't begin +// with prefix by replying with an HTTP 404 not found error. The prefix must +// match exactly: if the prefix in the request contains escaped characters +// the reply is also an HTTP 404 not found error. +func http_StripPrefix(prefix string, h http_Handler) http_Handler { if prefix == "" { return h } - return HandlerFunc(func(w ResponseWriter, r *Request) { - if p := strings.TrimPrefix(r.URL.Path, prefix); len(p) < len(r.URL.Path) { - r.URL.Path = p - h.ServeHTTP(w, r) + return http_HandlerFunc(func(w http_ResponseWriter, r *http_Request) { + p := strings.TrimPrefix(r.URL.Path, prefix) + rp := strings.TrimPrefix(r.URL.RawPath, prefix) + if len(p) < len(r.URL.Path) && (r.URL.RawPath == "" || len(rp) < len(r.URL.RawPath)) { + r2 := new(http_Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.Path = p + r2.URL.RawPath = rp + h.ServeHTTP(w, r2) } else { - NotFound(w, r) + http_NotFound(w, r) } }) } // Redirect replies to the request with a redirect to url, // which may be a path relative to the request path. -func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { - if u, err := url.Parse(urlStr); err == nil { +// Any non-ASCII characters in url will be percent-encoded, +// but existing percent encodings will not be changed. +// +// The provided code should be in the 3xx range and is usually +// [StatusMovedPermanently], [StatusFound] or [StatusSeeOther]. +// +// If the Content-Type header has not been set, [Redirect] sets it +// to "text/html; charset=utf-8" and writes a small HTML body. +// Setting the Content-Type header to any value, including nil, +// disables that behavior. +func http_Redirect(w http_ResponseWriter, r *http_Request, url string, code int) { + if u, err := urlpkg.Parse(url); err == nil { + // If url was relative, make its path absolute by + // combining with request path. + // The client would probably do this for us, + // but doing it ourselves is more reliable. + // See RFC 7231, section 7.1.2 + if u.Scheme == "" && u.Host == "" { + oldpath := r.URL.EscapedPath() + if oldpath == "" { // should not happen, but avoid a crash if it does + oldpath = "/" + } - oldpath := r.URL.Path - if oldpath == "" { - oldpath = "/" - } - if u.Scheme == "" { - - if urlStr == "" || urlStr[0] != '/' { - + // no leading http://server + if url == "" || url[0] != '/' { + // make relative path absolute olddir, _ := path.Split(oldpath) - urlStr = olddir + urlStr + url = olddir + url } var query string - if i := strings.Index(urlStr, "?"); i != -1 { - urlStr, query = urlStr[:i], urlStr[i:] + if i := strings.Index(url, "?"); i != -1 { + url, query = url[:i], url[i:] } - trailing := strings.HasSuffix(urlStr, "/") - urlStr = path.Clean(urlStr) - if trailing && !strings.HasSuffix(urlStr, "/") { - urlStr += "/" + // clean up but preserve trailing slash + trailing := strings.HasSuffix(url, "/") + url = path.Clean(url) + if trailing && !strings.HasSuffix(url, "/") { + url += "/" } - urlStr += query + url += query } } - w.Header().Set("Location", urlStr) + h := w.Header() + + // RFC 7231 notes that a short HTML body is usually included in + // the response because older user agents may not understand 301/307. + // Do it only if the request didn't already have a Content-Type header. + _, hadCT := h["Content-Type"] + + h.Set("Location", http_hexEscapeNonASCII(url)) + if !hadCT && (r.Method == "GET" || r.Method == "HEAD") { + h.Set("Content-Type", "text/html; charset=utf-8") + } w.WriteHeader(code) - if r.Method == "GET" { - note := "<a href=\"" + htmlEscape(urlStr) + "\">" + statusText[code] + "</a>.\n" - fmt.Fprintln(w, note) + // Shouldn't send the body for POST or HEAD; that leaves GET. + if !hadCT && r.Method == "GET" { + body := "<a href=\"" + http_htmlEscape(url) + "\">" + http_StatusText(code) + "</a>.\n" + fmt.Fprintln(w, body) } } -var htmlReplacer = strings.NewReplacer( +var http_htmlReplacer = strings.NewReplacer( "&", "&", "<", "<", ">", ">", - + // """ is shorter than """. ` + "`" + `"` + "`" + `, """, - + // "'" is shorter than "'" and apos was not in HTML until HTML5. "'", "'", ) -func htmlEscape(s string) string { - return htmlReplacer.Replace(s) +func http_htmlEscape(s string) string { + return http_htmlReplacer.Replace(s) } // Redirect to a fixed URL -type redirectHandler struct { +type http_redirectHandler struct { url string code int } -func (rh *redirectHandler) ServeHTTP(w ResponseWriter, r *Request) { - Redirect(w, r, rh.url, rh.code) +func (rh *http_redirectHandler) ServeHTTP(w http_ResponseWriter, r *http_Request) { + http_Redirect(w, r, rh.url, rh.code) } // RedirectHandler returns a request handler that redirects // each request it receives to the given url using the given // status code. -func RedirectHandler(url string, code int) Handler { - return &redirectHandler{url, code} +// +// The provided code should be in the 3xx range and is usually +// [StatusMovedPermanently], [StatusFound] or [StatusSeeOther]. +func http_RedirectHandler(url string, code int) http_Handler { + return &http_redirectHandler{url, code} } // ServeMux is an HTTP request multiplexer. @@ -4601,61 +10579,141 @@ // patterns and calls the handler for the pattern that // most closely matches the URL. // -// Patterns name fixed, rooted paths, like "/favicon.ico", -// or rooted subtrees, like "/images/" (note the trailing slash). -// Longer patterns take precedence over shorter ones, so that -// if there are handlers registered for both "/images/" -// and "/images/thumbnails/", the latter handler will be -// called for paths beginning "/images/thumbnails/" and the -// former will receive requests for any other paths in the -// "/images/" subtree. +// # Patterns // -// Note that since a pattern ending in a slash names a rooted subtree, -// the pattern "/" matches all paths not matched by other registered -// patterns, not just the URL with Path == "/". +// Patterns can match the method, host and path of a request. +// Some examples: // -// Patterns may optionally begin with a host name, restricting matches to -// URLs on that host only. Host-specific patterns take precedence over -// general patterns, so that a handler might register for the two patterns -// "/codesearch" and "codesearch.google.com/" without also taking over -// requests for "http://www.google.com/". +// - "/index.html" matches the path "/index.html" for any host and method. +// - "GET /static/" matches a GET request whose path begins with "/static/". +// - "example.com/" matches any request to the host "example.com". +// - "example.com/{$}" matches requests with host "example.com" and path "/". +// - "/b/{bucket}/o/{objectname...}" matches paths whose first segment is "b" +// and whose third segment is "o". The name "bucket" denotes the second +// segment and "objectname" denotes the remainder of the path. // -// ServeMux also takes care of sanitizing the URL request path, -// redirecting any request containing . or .. elements to an -// equivalent .- and ..-free URL. -type ServeMux struct { - mu sync.RWMutex - m map[string]muxEntry - hosts bool // whether any patterns contain hostnames +// In general, a pattern looks like +// +// [METHOD ][HOST]/[PATH] +// +// All three parts are optional; "/" is a valid pattern. +// If METHOD is present, it must be followed by at least one space or tab. +// +// Literal (that is, non-wildcard) parts of a pattern match +// the corresponding parts of a request case-sensitively. +// +// A pattern with no method matches every method. A pattern +// with the method GET matches both GET and HEAD requests. +// Otherwise, the method must match exactly. +// +// A pattern with no host matches every host. +// A pattern with a host matches URLs on that host only. +// +// A path can include wildcard segments of the form {NAME} or {NAME...}. +// For example, "/b/{bucket}/o/{objectname...}". +// The wildcard name must be a valid Go identifier. +// Wildcards must be full path segments: they must be preceded by a slash and followed by +// either a slash or the end of the string. +// For example, "/b_{bucket}" is not a valid pattern. +// +// Normally a wildcard matches only a single path segment, +// ending at the next literal slash (not %2F) in the request URL. +// But if the "..." is present, then the wildcard matches the remainder of the URL path, including slashes. +// (Therefore it is invalid for a "..." wildcard to appear anywhere but at the end of a pattern.) +// The match for a wildcard can be obtained by calling [Request.PathValue] with the wildcard's name. +// A trailing slash in a path acts as an anonymous "..." wildcard. +// +// The special wildcard {$} matches only the end of the URL. +// For example, the pattern "/{$}" matches only the path "/", +// whereas the pattern "/" matches every path. +// +// For matching, both pattern paths and incoming request paths are unescaped segment by segment. +// So, for example, the path "/a%2Fb/100%25" is treated as having two segments, "a/b" and "100%". +// The pattern "/a%2fb/" matches it, but the pattern "/a/b/" does not. +// +// # Precedence +// +// If two or more patterns match a request, then the most specific pattern takes precedence. +// A pattern P1 is more specific than P2 if P1 matches a strict subset of P2’s requests; +// that is, if P2 matches all the requests of P1 and more. +// If neither is more specific, then the patterns conflict. +// There is one exception to this rule, for backwards compatibility: +// if two patterns would otherwise conflict and one has a host while the other does not, +// then the pattern with the host takes precedence. +// If a pattern passed to [ServeMux.Handle] or [ServeMux.HandleFunc] conflicts with +// another pattern that is already registered, those functions panic. +// +// As an example of the general rule, "/images/thumbnails/" is more specific than "/images/", +// so both can be registered. +// The former matches paths beginning with "/images/thumbnails/" +// and the latter will match any other path in the "/images/" subtree. +// +// As another example, consider the patterns "GET /" and "/index.html": +// both match a GET request for "/index.html", but the former pattern +// matches all other GET and HEAD requests, while the latter matches any +// request for "/index.html" that uses a different method. +// The patterns conflict. +// +// # Trailing-slash redirection +// +// Consider a [ServeMux] with a handler for a subtree, registered using a trailing slash or "..." wildcard. +// If the ServeMux receives a request for the subtree root without a trailing slash, +// it redirects the request by adding the trailing slash. +// This behavior can be overridden with a separate registration for the path without +// the trailing slash or "..." wildcard. For example, registering "/images/" causes ServeMux +// to redirect a request for "/images" to "/images/", unless "/images" has +// been registered separately. +// +// # Request sanitizing +// +// ServeMux also takes care of sanitizing the URL request path and the Host +// header, stripping the port number and redirecting any request containing . or +// .. segments or repeated slashes to an equivalent, cleaner URL. +// Escaped path elements such as "%2e" for "." and "%2f" for "/" are preserved +// and aren't considered separators for request routing. +// +// # Compatibility +// +// The pattern syntax and matching behavior of ServeMux changed significantly +// in Go 1.22. To restore the old behavior, set the GODEBUG environment variable +// to "httpmuxgo121=1". This setting is read once, at program startup; changes +// during execution will be ignored. +// +// The backwards-incompatible changes include: +// - Wildcards are just ordinary literal path segments in 1.21. +// For example, the pattern "/{x}" will match only that path in 1.21, +// but will match any one-segment path in 1.22. +// - In 1.21, no pattern was rejected, unless it was empty or conflicted with an existing pattern. +// In 1.22, syntactically invalid patterns will cause [ServeMux.Handle] and [ServeMux.HandleFunc] to panic. +// For example, in 1.21, the patterns "/{" and "/a{x}" match themselves, +// but in 1.22 they are invalid and will cause a panic when registered. +// - In 1.22, each segment of a pattern is unescaped; this was not done in 1.21. +// For example, in 1.22 the pattern "/%61" matches the path "/a" ("%61" being the URL escape sequence for "a"), +// but in 1.21 it would match only the path "/%2561" (where "%25" is the escape for the percent sign). +// - When matching patterns to paths, in 1.22 each segment of the path is unescaped; in 1.21, the entire path is unescaped. +// This change mostly affects how paths with %2F escapes adjacent to slashes are treated. +// See https://go.dev/issue/21955 for details. +type http_ServeMux struct { + mu sync.RWMutex + tree http_routingNode + index http_routingIndex + mux121 http_serveMux121 // used only when GODEBUG=httpmuxgo121=1 } -type muxEntry struct { - explicit bool - h Handler - pattern string +// NewServeMux allocates and returns a new [ServeMux]. +func http_NewServeMux() *http_ServeMux { + return &http_ServeMux{} } -// NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return &ServeMux{m: make(map[string]muxEntry)} } +// DefaultServeMux is the default [ServeMux] used by [Serve]. +var http_DefaultServeMux = &http_defaultServeMux -// DefaultServeMux is the default ServeMux used by Serve. -var DefaultServeMux = NewServeMux() +var http_defaultServeMux http_ServeMux -// Does path match pattern? -func pathMatch(pattern, path string) bool { - if len(pattern) == 0 { +// cleanPath returns the canonical path for p, eliminating . and .. elements. - return false - } - n := len(pattern) - if pattern[n-1] != '/' { - return pattern == path - } - return len(path) >= n && path[0:n] == pattern -} - -// Return the canonical path for p, eliminating . and .. elements. -func cleanPath(p string) string { +// cleanPath returns the canonical path for p, eliminating . and .. elements. +func http_cleanPath(p string) string { if p == "" { return "/" } @@ -4663,191 +10721,700 @@ p = "/" + p } np := path.Clean(p) - + // path.Clean removes trailing slash except for root; + // put the trailing slash back if necessary. if p[len(p)-1] == '/' && np != "/" { - np += "/" + // Fast path for common case of p being the string we want: + if len(p) == len(np)+1 && strings.HasPrefix(p, np) { + np = p + } else { + np += "/" + } } return np } -// Find a handler on a handler map given a path string -// Most-specific (longest) pattern wins -func (mux *ServeMux) match(path string) (h Handler, pattern string) { - var n = 0 - for k, v := range mux.m { - if !pathMatch(k, path) { - continue - } - if h == nil || len(k) > n { - n = len(k) - h = v.h - pattern = v.pattern - } +// stripHostPort returns h without any trailing ":<port>". +func http_stripHostPort(h string) string { + // If no port on host, return unchanged + if !strings.Contains(h, ":") { + return h } - return + host, _, err := net.SplitHostPort(h) + if err != nil { + return h // on error, return unchanged + } + return host } // Handler returns the handler to use for the given request, // consulting r.Method, r.Host, and r.URL.Path. It always returns // a non-nil handler. If the path is not in its canonical form, the // handler will be an internally-generated handler that redirects -// to the canonical path. +// to the canonical path. If the host contains a port, it is ignored +// when matching handlers. +// +// The path and host are used unchanged for CONNECT requests. // // Handler also returns the registered pattern that matches the // request or, in the case of internally-generated redirects, -// the pattern that will match after following the redirect. +// the path that will match after following the redirect. // // If there is no registered handler that applies to the request, -// Handler returns a ` + "`" + `` + "`" + `page not found'' handler and an empty pattern. -func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { - if r.Method != "CONNECT" { - if p := cleanPath(r.URL.Path); p != r.URL.Path { - _, pattern = mux.handler(r.Host, p) - url := *r.URL - url.Path = p - return RedirectHandler(url.String(), StatusMovedPermanently), pattern - } +// Handler returns a “page not found” or “method not supported” +// handler and an empty pattern. +// +// Handler does not modify its argument. In particular, it does not +// populate named path wildcards, so r.PathValue will always return +// the empty string. +func (mux *http_ServeMux) Handler(r *http_Request) (h http_Handler, pattern string) { + if http_use121 { + return mux.mux121.findHandler(r) } - - return mux.handler(r.Host, r.URL.Path) + h, p, _, _ := mux.findHandler(r) + return h, p } -// handler is the main implementation of Handler. -// The path is known to be in canonical form, except for CONNECT methods. -func (mux *ServeMux) handler(host, path string) (h Handler, pattern string) { +// findHandler finds a handler for a request. +// If there is a matching handler, it returns it and the pattern that matched. +// Otherwise it returns a Redirect or NotFound handler with the path that would match +// after the redirect. +func (mux *http_ServeMux) findHandler(r *http_Request) (h http_Handler, patStr string, _ *http_pattern, matches []string) { + var n *http_routingNode + host := r.URL.Host + escapedPath := r.URL.EscapedPath() + path := escapedPath + // CONNECT requests are not canonicalized. + if r.Method == "CONNECT" { + // If r.URL.Path is /tree and its handler is not registered, + // the /tree -> /tree/ redirect applies to CONNECT requests + // but the path canonicalization does not. + _, _, u := mux.matchOrRedirect(host, r.Method, path, r.URL) + if u != nil { + return http_RedirectHandler(u.String(), http_StatusTemporaryRedirect), u.Path, nil, nil + } + // Redo the match, this time with r.Host instead of r.URL.Host. + // Pass a nil URL to skip the trailing-slash redirect logic. + n, matches, _ = mux.matchOrRedirect(r.Host, r.Method, path, nil) + } else { + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host = http_stripHostPort(r.Host) + path = http_cleanPath(path) + + // If the given path is /tree and its handler is not registered, + // redirect for /tree/. + var u *url.URL + n, matches, u = mux.matchOrRedirect(host, r.Method, path, r.URL) + if u != nil { + return http_RedirectHandler(u.String(), http_StatusTemporaryRedirect), n.pattern.String(), nil, nil + } + if path != escapedPath { + // Redirect to cleaned path. + patStr := "" + if n != nil { + patStr = n.pattern.String() + } + u := http_urlFromEscaped(path, r.URL.RawQuery) + return http_RedirectHandler(u.String(), http_StatusTemporaryRedirect), patStr, nil, nil + } + } + if n == nil { + // We didn't find a match with the request method. To distinguish between + // Not Found and Method Not Allowed, see if there is another pattern that + // matches except for the method. + allowedMethods := mux.matchingMethods(host, path) + if len(allowedMethods) > 0 { + return http_HandlerFunc(func(w http_ResponseWriter, r *http_Request) { + w.Header().Set("Allow", strings.Join(allowedMethods, ", ")) + http_Error(w, http_StatusText(http_StatusMethodNotAllowed), http_StatusMethodNotAllowed) + }), "", nil, nil + } + return http_NotFoundHandler(), "", nil, nil + } + return n.handler, n.pattern.String(), n.pattern, matches +} + +// matchOrRedirect looks up a node in the tree that matches the host, method and path. +// +// If the url argument is non-nil, handler also deals with trailing-slash +// redirection: when a path doesn't match exactly, the match is tried again +// after appending "/" to the path. If that second match succeeds, the last +// return value is the URL to redirect to. +func (mux *http_ServeMux) matchOrRedirect(host, method, path string, u *url.URL) (_ *http_routingNode, matches []string, redirectTo *url.URL) { mux.mu.RLock() defer mux.mu.RUnlock() - if mux.hosts { - h, pattern = mux.match(host + path) + n, matches := mux.tree.match(host, method, path) + // We can terminate here if any of the following is true: + // - We have an exact match already. + // - We were asked not to try trailing slash redirection. + // - The URL already has a trailing slash. + // - The URL is an empty string. + if !http_exactMatch(n, path) && u != nil && !strings.HasSuffix(path, "/") && path != "" { + // If there is an exact match with a trailing slash, then redirect. + path += "/" + n2, _ := mux.tree.match(host, method, path) + if http_exactMatch(n2, path) { + // It is safe to return n2 here: it is used only in the second RedirectHandler case + // of findHandler, and that method returns before it does the "n == nil" check where + // the first return value matters. We return it here only to make the pattern available + // to findHandler. + return n2, nil, http_urlFromEscaped(path, u.RawQuery) + } } - if h == nil { - h, pattern = mux.match(path) + return n, matches, nil +} + +// urlFromEscaped returns a url.URL constructed from an escaped path and a raw +// query. +// +// It ensures that the Path and RawPath fields are in sync by unescaping the +// escaped path. Populating only the Path field and leaving RawPath empty (or +// failing to keep them in sync) can cause url.URL.String to produce a URL with +// either unexpected escaping (e.g., double-escaping "%" into "%25" in an +// already escaped path) or a lack thereof (e.g., losing the escaping of "%2f" +// and turning it into a literal path separator "/"). +func http_urlFromEscaped(escaped, rawQuery string) *url.URL { + unescaped, err := url.PathUnescape(escaped) + // Should be impossible, since ServeMux will reject unparsable URLs way + // earlier. + if err != nil { + unescaped = escaped } - if h == nil { - h, pattern = NotFoundHandler(), "" + return &url.URL{ + Path: unescaped, + RawPath: escaped, + RawQuery: rawQuery, } - return +} + +// exactMatch reports whether the node's pattern exactly matches the path. +// As a special case, if the node is nil, exactMatch return false. +// +// Before wildcards were introduced, it was clear that an exact match meant +// that the pattern and path were the same string. The only other possibility +// was that a trailing-slash pattern, like "/", matched a path longer than +// it, like "/a". +// +// With wildcards, we define an inexact match as any one where a multi wildcard +// matches a non-empty string. All other matches are exact. +// For example, these are all exact matches: +// +// pattern path +// /a /a +// /{x} /a +// /a/{$} /a/ +// /a/ /a/ +// +// The last case has a multi wildcard (implicitly), but the match is exact because +// the wildcard matches the empty string. +// +// Examples of matches that are not exact: +// +// pattern path +// / /a +// /a/{x...} /a/b +func http_exactMatch(n *http_routingNode, path string) bool { + if n == nil { + return false + } + // We can't directly implement the definition (empty match for multi + // wildcard) because we don't record a match for anonymous multis. + + // If there is no multi, the match is exact. + if !n.pattern.lastSegment().multi { + return true + } + + // If the path doesn't end in a trailing slash, then the multi match + // is non-empty. + if len(path) > 0 && path[len(path)-1] != '/' { + return false + } + // Only patterns ending in {$} or a multi wildcard can + // match a path with a trailing slash. + // For the match to be exact, the number of pattern + // segments should be the same as the number of slashes in the path. + // E.g. "/a/b/{$}" and "/a/b/{...}" exactly match "/a/b/", but "/a/" does not. + return len(n.pattern.segments) == strings.Count(path, "/") +} + +// matchingMethods return a sorted list of all methods that would match with the given host and path. +func (mux *http_ServeMux) matchingMethods(host, path string) []string { + // Hold the read lock for the entire method so that the two matches are done + // on the same set of registered patterns. + mux.mu.RLock() + defer mux.mu.RUnlock() + ms := map[string]bool{} + mux.tree.matchingMethods(host, path, ms) + // matchOrRedirect will try appending a trailing slash if there is no match. + if !strings.HasSuffix(path, "/") { + mux.tree.matchingMethods(host, path+"/", ms) + } + return slices.Sorted(maps.Keys(ms)) } // ServeHTTP dispatches the request to the handler whose // pattern most closely matches the request URL. -func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { +func (mux *http_ServeMux) ServeHTTP(w http_ResponseWriter, r *http_Request) { if r.RequestURI == "*" { if r.ProtoAtLeast(1, 1) { w.Header().Set("Connection", "close") } - w.WriteHeader(StatusBadRequest) + w.WriteHeader(http_StatusBadRequest) return } - h, _ := mux.Handler(r) + var h http_Handler + if http_use121 { + h, _ = mux.mux121.findHandler(r) + } else { + h, r.Pattern, r.pat, r.matches = mux.findHandler(r) + } h.ServeHTTP(w, r) } +// The four functions below all call ServeMux.register so that callerLocation +// always refers to user code. + // Handle registers the handler for the given pattern. -// If a handler already exists for pattern, Handle panics. -func (mux *ServeMux) Handle(pattern string, handler Handler) { - mux.mu.Lock() - defer mux.mu.Unlock() - - if pattern == "" { - panic("http: invalid pattern " + pattern) - } - if handler == nil { - panic("http: nil handler") - } - if mux.m[pattern].explicit { - panic("http: multiple registrations for " + pattern) - } - - mux.m[pattern] = muxEntry{explicit: true, h: handler, pattern: pattern} - - if pattern[0] != '/' { - mux.hosts = true - } - - n := len(pattern) - if n > 0 && pattern[n-1] == '/' && !mux.m[pattern[0:n-1]].explicit { - - path := pattern - if pattern[0] != '/' { - - path = pattern[strings.Index(pattern, "/"):] - } - url := &url.URL{Path: path} - mux.m[pattern[0:n-1]] = muxEntry{h: RedirectHandler(url.String(), StatusMovedPermanently), pattern: pattern} +// If the given pattern conflicts with one that is already registered +// or if the pattern is invalid, Handle panics. +// +// See [ServeMux] for details on valid patterns and conflict rules. +func (mux *http_ServeMux) Handle(pattern string, handler http_Handler) { + if http_use121 { + mux.mux121.handle(pattern, handler) + } else { + mux.register(pattern, handler) } } // HandleFunc registers the handler function for the given pattern. -func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { - mux.Handle(pattern, HandlerFunc(handler)) +// If the given pattern conflicts with one that is already registered +// or if the pattern is invalid, HandleFunc panics. +// +// See [ServeMux] for details on valid patterns and conflict rules. +func (mux *http_ServeMux) HandleFunc(pattern string, handler func(http_ResponseWriter, *http_Request)) { + if http_use121 { + mux.mux121.handleFunc(pattern, handler) + } else { + mux.register(pattern, http_HandlerFunc(handler)) + } } -// Handle registers the handler for the given pattern -// in the DefaultServeMux. -// The documentation for ServeMux explains how patterns are matched. -func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } +// Handle registers the handler for the given pattern in [DefaultServeMux]. +// The documentation for [ServeMux] explains how patterns are matched. +func http_Handle(pattern string, handler http_Handler) { + if http_use121 { + http_DefaultServeMux.mux121.handle(pattern, handler) + } else { + http_DefaultServeMux.register(pattern, handler) + } +} -// HandleFunc registers the handler function for the given pattern -// in the DefaultServeMux. -// The documentation for ServeMux explains how patterns are matched. -func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { - DefaultServeMux.HandleFunc(pattern, handler) +// HandleFunc registers the handler function for the given pattern in [DefaultServeMux]. +// The documentation for [ServeMux] explains how patterns are matched. +func http_HandleFunc(pattern string, handler func(http_ResponseWriter, *http_Request)) { + if http_use121 { + http_DefaultServeMux.mux121.handleFunc(pattern, handler) + } else { + http_DefaultServeMux.register(pattern, http_HandlerFunc(handler)) + } +} + +func (mux *http_ServeMux) register(pattern string, handler http_Handler) { + if err := mux.registerErr(pattern, handler); err != nil { + panic(err) + } +} + +func (mux *http_ServeMux) registerErr(patstr string, handler http_Handler) error { + if patstr == "" { + return errors.New("http: invalid pattern") + } + if handler == nil { + return errors.New("http: nil handler") + } + if f, ok := handler.(http_HandlerFunc); ok && f == nil { + return errors.New("http: nil handler") + } + + pat, err := http_parsePattern(patstr) + if err != nil { + return fmt.Errorf("parsing %q: %w", patstr, err) + } + + // Get the caller's location, for better conflict error messages. + // Skip register and whatever calls it. + _, file, line, ok := runtime.Caller(3) + if !ok { + pat.loc = "unknown location" + } else { + pat.loc = fmt.Sprintf("%s:%d", file, line) + } + + mux.mu.Lock() + defer mux.mu.Unlock() + // Check for conflict. + if err := mux.index.possiblyConflictingPatterns(pat, func(pat2 *http_pattern) error { + if pat.conflictsWith(pat2) { + d := http_describeConflict(pat, pat2) + return fmt.Errorf("pattern %q (registered at %s) conflicts with pattern %q (registered at %s):\n%s", + pat, pat.loc, pat2, pat2.loc, d) + } + return nil + }); err != nil { + return err + } + mux.tree.addPattern(pat, handler) + mux.index.addPattern(pat) + return nil } // Serve accepts incoming HTTP connections on the listener l, -// creating a new service goroutine for each. The service goroutines +// creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. -// Handler is typically nil, in which case the DefaultServeMux is used. -func Serve(l net.Listener, handler Handler) error { - srv := &Server{Handler: handler} +// +// The handler is typically nil, in which case [DefaultServeMux] is used. +// +// HTTP/2 support is only enabled if the Listener returns [*tls.Conn] +// connections or connections which implement the same ConnectionState +// method as *tls.Conn, and the connection state indicates that the "h2" +// protocol was negotiated by ALPN. +// +// Serve always returns a non-nil error. +func http_Serve(l net.Listener, handler http_Handler) error { + srv := &http_Server{Handler: handler} return srv.Serve(l) } +// ServeTLS accepts incoming HTTPS connections on the listener l, +// creating a new service goroutine for each. The service goroutines +// read requests and then call handler to reply to them. +// +// The handler is typically nil, in which case [DefaultServeMux] is used. +// +// Additionally, files containing a certificate and matching private key +// for the server must be provided. If the certificate is signed by a +// certificate authority, the certFile should be the concatenation +// of the server's certificate, any intermediates, and the CA's certificate. +// +// ServeTLS always returns a non-nil error. +func http_ServeTLS(l net.Listener, handler http_Handler, certFile, keyFile string) error { + srv := &http_Server{Handler: handler} + return srv.ServeTLS(l, certFile, keyFile) +} + // A Server defines parameters for running an HTTP server. // The zero value for Server is a valid configuration. -type Server struct { - Addr string // TCP address to listen on, ":http" if empty - Handler Handler // handler to invoke, http.DefaultServeMux if nil - ReadTimeout time.Duration // maximum duration before timing out read of the request - WriteTimeout time.Duration // maximum duration before timing out write of the response - MaxHeaderBytes int // maximum size of request headers, DefaultMaxHeaderBytes if 0 - TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS +type http_Server struct { + // Addr optionally specifies the TCP address for the server to listen on, + // in the form "host:port". If empty, ":http" (port 80) is used. + // The service names are defined in RFC 6335 and assigned by IANA. + // See net.Dial for details of the address format. + Addr string + + Handler http_Handler // handler to invoke, http.DefaultServeMux if nil + + // DisableGeneralOptionsHandler, if true, passes "OPTIONS *" requests to the Handler, + // otherwise responds with 200 OK and Content-Length: 0. + DisableGeneralOptionsHandler bool + + // TLSConfig optionally provides a TLS configuration for use + // by ServeTLS and ListenAndServeTLS. Note that this value is + // cloned by ServeTLS and ListenAndServeTLS, so it's not + // possible to modify the configuration with methods like + // tls.Config.SetSessionTicketKeys. To use + // SetSessionTicketKeys, use Server.Serve with a TLS Listener + // instead. + TLSConfig *tls.Config + + // ReadTimeout is the maximum duration for reading the entire + // request, including the body. A zero or negative value means + // there will be no timeout. + // + // Because ReadTimeout does not let Handlers make per-request + // decisions on each request body's acceptable deadline or + // upload rate, most users will prefer to use + // ReadHeaderTimeout. It is valid to use them both. + ReadTimeout time.Duration + + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If zero, the value of + // ReadTimeout is used. If negative, or if zero and ReadTimeout + // is zero or negative, there is no timeout. + ReadHeaderTimeout time.Duration + + // WriteTimeout is the maximum duration before timing out + // writes of the response. It is reset whenever a new + // request's header is read. Like ReadTimeout, it does not + // let Handlers make decisions on a per-request basis. + // A zero or negative value means there will be no timeout. + WriteTimeout time.Duration + + // IdleTimeout is the maximum amount of time to wait for the + // next request when keep-alives are enabled. If zero, the value + // of ReadTimeout is used. If negative, or if zero and ReadTimeout + // is zero or negative, there is no timeout. + IdleTimeout time.Duration + + // MaxHeaderBytes controls the maximum number of bytes the + // server will read parsing the request header's keys and + // values, including the request line. It does not limit the + // size of the request body. + // If zero, DefaultMaxHeaderBytes is used. + MaxHeaderBytes int + + // MaxHeaderValueCount controls the maximum number of header + // values that the server is willing to parse from a request. + // If zero, DefaultMaxHeaderValueCount is used. + // Note that comma-separated values in a single header line are + // counted once, while values sent as multiple header lines are + // counted multiple times. + MaxHeaderValueCount int // TLSNextProto optionally specifies a function to take over - // ownership of the provided TLS connection when an NPN - // protocol upgrade has occurred. The map key is the protocol + // ownership of the provided TLS connection when an ALPN + // protocol upgrade has occurred. The map key is the protocol // name negotiated. The Handler argument should be used to // handle HTTP requests and will initialize the Request's TLS - // and RemoteAddr if not already set. The connection is + // and RemoteAddr if not already set. The connection is // automatically closed when the function returns. - TLSNextProto map[string]func(*Server, *tls.Conn, Handler) + // If TLSNextProto is not nil, HTTP/2 support is not enabled + // automatically. + // + // Historically, TLSNextProto was used to disable HTTP/2 support. + // The Server.Protocols field now provides a simpler way to do this. + TLSNextProto map[string]func(*http_Server, *tls.Conn, http_Handler) // ConnState specifies an optional callback function that is // called when a client connection changes state. See the // ConnState type and associated constants for details. - ConnState func(net.Conn, ConnState) + ConnState func(net.Conn, http_ConnState) // ErrorLog specifies an optional logger for errors accepting - // connections and unexpected behavior from handlers. - // If nil, logging goes to os.Stderr via the log package's - // standard logger. + // connections, unexpected behavior from handlers, and + // underlying FileSystem errors. + // If nil, logging is done via the log package's standard logger. ErrorLog *log.Logger - disableKeepAlives int32 // accessed atomically. + // BaseContext optionally specifies a function that returns + // the base context for incoming requests on this server. + // The provided Listener is the specific Listener that's + // about to start accepting requests. + // If BaseContext is nil, the default is context.Background(). + // If non-nil, it must return a non-nil context. + BaseContext func(net.Listener) context.Context + + // ConnContext optionally specifies a function that modifies + // the context used for a new connection c. The provided ctx + // is derived from the base context and has a ServerContextKey + // value. + ConnContext func(ctx context.Context, c net.Conn) context.Context + + // HTTP2 configures HTTP/2 connections. + HTTP2 *http_HTTP2Config + + // Protocols is the set of protocols accepted by the server. + // + // If Protocols includes UnencryptedHTTP2, the server will accept + // unencrypted HTTP/2 connections. The server can serve both + // HTTP/1 and unencrypted HTTP/2 on the same address and port. + // + // If Protocols is nil, the default is usually HTTP/1 and HTTP/2. + // If TLSNextProto is non-nil and does not contain an "h2" entry, + // the default is HTTP/1 only. + Protocols *http_Protocols + + // DisableClientPriority specifies whether client-specified priority, as + // specified in RFC 9218, should be respected or not. + // + // This field only takes effect if using HTTP/2, and if no custom write + // scheduler is defined for the HTTP/2 server. Otherwise, this field is a + // no-op. + // + // If set to true, requests will be served in a round-robin manner, without + // prioritization. + DisableClientPriority bool + + inShutdown atomic.Bool // true when server is in shutdown + + disableKeepAlives atomic.Bool + nextProtoOnce sync.Once // guards setupHTTP2_* init + nextProtoErr error // result of http2.ConfigureServer if used + + mu sync.Mutex + listeners map[*net.Listener]struct{} + activeConn map[*http_conn]struct{} + onShutdown []func() + h2 *http_http2Server + h2Config http_http2ExternalServerConfig + h2IdleTimeout time.Duration + h3 *http_http3ServerHandler + + listenerGroup sync.WaitGroup +} + +// Close immediately closes all active net.Listeners and any +// connections in state [StateNew], [StateActive], or [StateIdle]. For a +// graceful shutdown, use [Server.Shutdown]. +// +// Close does not attempt to close (and does not even know about) +// any hijacked connections, such as WebSockets. +// +// Close returns any error returned from closing the [Server]'s +// underlying Listener(s). +func (s *http_Server) Close() error { + s.inShutdown.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + err := s.closeListenersLocked() + + // Unlock s.mu while waiting for listenerGroup. + // The group Add and Done calls are made with s.mu held, + // to avoid adding a new listener in the window between + // us setting inShutdown above and waiting here. + s.mu.Unlock() + s.listenerGroup.Wait() + s.mu.Lock() + + for c := range s.activeConn { + c.rwc.Close() + delete(s.activeConn, c) + } + return err +} + +// shutdownPollIntervalMax is the max polling interval when checking +// quiescence during Server.Shutdown. Polling starts with a small +// interval and backs off to the max. +// Ideally we could find a solution that doesn't involve polling, +// but which also doesn't have a high runtime cost (and doesn't +// involve any contentious mutexes), but that is left as an +// exercise for the reader. +const http_shutdownPollIntervalMax = 500 * time.Millisecond + +// Shutdown gracefully shuts down the server without interrupting any +// active connections. Shutdown works by first closing all open +// listeners, then closing all idle connections, and then waiting +// indefinitely for connections to return to idle and then shut down. +// If the provided context expires before the shutdown is complete, +// Shutdown returns the context's error, otherwise it returns any +// error returned from closing the [Server]'s underlying Listener(s). +// +// When Shutdown is called, [Serve], [ServeTLS], [ListenAndServe], and +// [ListenAndServeTLS] immediately return [ErrServerClosed]. Make sure the +// program doesn't exit and waits instead for Shutdown to return. +// +// Shutdown does not attempt to close nor wait for hijacked +// connections such as WebSockets. The caller of Shutdown should +// separately notify such long-lived connections of shutdown and wait +// for them to close, if desired. See [Server.RegisterOnShutdown] for a way to +// register shutdown notification functions. +// +// Once Shutdown has been called on a server, it may not be reused; +// future calls to methods such as Serve will return ErrServerClosed. +func (s *http_Server) Shutdown(ctx context.Context) error { + s.inShutdown.Store(true) + + s.mu.Lock() + if s.h3 != nil { + s.h3.shutdownCtx = ctx + } + lnerr := s.closeListenersLocked() + for _, f := range s.onShutdown { + go f() + } + s.mu.Unlock() + s.listenerGroup.Wait() + + pollIntervalBase := time.Millisecond + nextPollInterval := func() time.Duration { + // Add 10% jitter. + interval := pollIntervalBase + time.Duration(rand.IntN(int(pollIntervalBase/10))) + // Double and clamp for next time. + pollIntervalBase *= 2 + if pollIntervalBase > http_shutdownPollIntervalMax { + pollIntervalBase = http_shutdownPollIntervalMax + } + return interval + } + + timer := time.NewTimer(nextPollInterval()) + defer timer.Stop() + for { + if s.closeIdleConns() { + return lnerr + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + timer.Reset(nextPollInterval()) + } + } +} + +// RegisterOnShutdown registers a function to call on [Server.Shutdown]. +// This can be used to gracefully shutdown connections that have +// undergone ALPN protocol upgrade or that have been hijacked. +// This function should start protocol-specific graceful shutdown, +// but should not wait for shutdown to complete. +func (s *http_Server) RegisterOnShutdown(f func()) { + s.mu.Lock() + s.onShutdown = append(s.onShutdown, f) + s.mu.Unlock() +} + +// closeIdleConns closes all idle connections and reports whether the +// server is quiescent. +func (s *http_Server) closeIdleConns() bool { + s.mu.Lock() + defer s.mu.Unlock() + quiescent := true + for c := range s.activeConn { + st, unixSec := c.getState() + // Issue 22682: treat StateNew connections as if + // they're idle if we haven't read the first request's + // header in over 5 seconds. + if st == http_StateNew && unixSec < time.Now().Unix()-5 { + st = http_StateIdle + } + if st != http_StateIdle || unixSec == 0 { + // Assume unixSec == 0 means it's a very new + // connection, without state set yet. + quiescent = false + continue + } + c.rwc.Close() + delete(s.activeConn, c) + } + return quiescent +} + +func (s *http_Server) closeListenersLocked() error { + var err error + for ln := range s.listeners { + if cerr := (*ln).Close(); cerr != nil && err == nil { + err = cerr + } + } + return err } // A ConnState represents the state of a client connection to a server. -// It's used by the optional Server.ConnState hook. -type ConnState int +// It's used by the optional [Server.ConnState] hook. +type http_ConnState int const ( // StateNew represents a new connection that is expected to // send a request immediately. Connections begin at this // state and then transition to either StateActive or // StateClosed. - StateNew ConnState = iota + http_StateNew http_ConnState = iota // StateActive represents a connection that has read 1 or more // bytes of a request. The Server.ConnState hook for @@ -4855,58 +11422,107 @@ // and doesn't fire again until the request has been // handled. After the request is handled, the state // transitions to StateClosed, StateHijacked, or StateIdle. - StateActive + // For HTTP/2, StateActive fires on the transition from zero + // to one active request, and only transitions away once all + // active requests are complete. That means that ConnState + // cannot be used to do per-request work; ConnState only notes + // the overall state of the connection. + http_StateActive // StateIdle represents a connection that has finished // handling a request and is in the keep-alive state, waiting // for a new request. Connections transition from StateIdle // to either StateActive or StateClosed. - StateIdle + http_StateIdle // StateHijacked represents a hijacked connection. // This is a terminal state. It does not transition to StateClosed. - StateHijacked + http_StateHijacked // StateClosed represents a closed connection. // This is a terminal state. Hijacked connections do not // transition to StateClosed. - StateClosed + http_StateClosed ) -var stateName = map[ConnState]string{ - StateNew: "new", - StateActive: "active", - StateIdle: "idle", - StateHijacked: "hijacked", - StateClosed: "closed", +var http_stateName = map[http_ConnState]string{ + http_StateNew: "new", + http_StateActive: "active", + http_StateIdle: "idle", + http_StateHijacked: "hijacked", + http_StateClosed: "closed", } -func (c ConnState) String() string { - return stateName[c] +func (c http_ConnState) String() string { + return http_stateName[c] } // serverHandler delegates to either the server's Handler or // DefaultServeMux and also handles "OPTIONS *" requests. -type serverHandler struct { - srv *Server +type http_serverHandler struct { + srv *http_Server } -func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { +// ServeHTTP should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/erda-project/erda-infra +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname badServeHTTP net/http.serverHandler.ServeHTTP +func (sh http_serverHandler) ServeHTTP(rw http_ResponseWriter, req *http_Request) { handler := sh.srv.Handler if handler == nil { - handler = DefaultServeMux + handler = http_DefaultServeMux } - if req.RequestURI == "*" && req.Method == "OPTIONS" { - handler = globalOptionsHandler{} + if !sh.srv.DisableGeneralOptionsHandler && req.RequestURI == "*" && req.Method == "OPTIONS" { + handler = http_globalOptionsHandler{} } + handler.ServeHTTP(rw, req) } -// ListenAndServe listens on the TCP network address srv.Addr and then -// calls Serve to handle requests on incoming connections. If -// srv.Addr is blank, ":http" is used. -func (srv *Server) ListenAndServe() error { - addr := srv.Addr +func http_badServeHTTP(http_serverHandler, http_ResponseWriter, *http_Request) + +// AllowQuerySemicolons returns a handler that serves requests by converting any +// unescaped semicolons in the URL query to ampersands, and invoking the handler h. +// +// This restores the pre-Go 1.17 behavior of splitting query parameters on both +// semicolons and ampersands. (See golang.org/issue/25192). Note that this +// behavior doesn't match that of many proxies, and the mismatch can lead to +// security issues. +// +// AllowQuerySemicolons should be invoked before [Request.ParseForm] is called. +func http_AllowQuerySemicolons(h http_Handler) http_Handler { + return http_HandlerFunc(func(w http_ResponseWriter, r *http_Request) { + if strings.Contains(r.URL.RawQuery, ";") { + r2 := new(http_Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.RawQuery = strings.ReplaceAll(r.URL.RawQuery, ";", "&") + h.ServeHTTP(w, r2) + } else { + h.ServeHTTP(w, r) + } + }) +} + +// ListenAndServe listens on the TCP network address s.Addr and then +// calls [Serve] to handle requests on incoming connections. +// Accepted connections are configured to enable TCP keep-alives. +// +// If s.Addr is blank, ":http" is used. +// +// ListenAndServe always returns a non-nil error. After [Server.Shutdown] or [Server.Close], +// the returned error is [ErrServerClosed]. +func (s *http_Server) ListenAndServe() error { + if s.shuttingDown() { + return http_ErrServerClosed + } + addr := s.Addr if addr == "" { addr = ":http" } @@ -4914,19 +11530,101 @@ if err != nil { return err } - return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)}) + return s.Serve(ln) } +var http_testHookServerServe func(*http_Server, net.Listener) // used if non-nil + +// shouldConfigureHTTP2ForServe reports whether Server.Serve should configure +// automatic HTTP/2. (which sets up the s.TLSNextProto map) +func (s *http_Server) shouldConfigureHTTP2ForServe() bool { + if s.TLSConfig == nil { + // Compatibility with Go 1.6: + // If there's no TLSConfig, it's possible that the user just + // didn't set it on the http.Server, but did pass it to + // tls.NewListener and passed that listener to Serve. + // So we should configure HTTP/2 (to set up s.TLSNextProto) + // in case the listener returns an "h2" *tls.Conn. + return true + } + if s.protocols().UnencryptedHTTP2() { + return true + } + // The user specified a TLSConfig on their http.Server. + // In this, case, only configure HTTP/2 if their tls.Config + // explicitly mentions "h2". Otherwise http2.ConfigureServer + // would modify the tls.Config to add it, but they probably already + // passed this tls.Config to tls.NewListener. And if they did, + // it's too late anyway to fix it. It would only be potentially racy. + // See Issue 15908. + return slices.Contains(s.TLSConfig.NextProtos, "h2") +} + +// ErrServerClosed is returned by the [Server.Serve], [ServeTLS], [ListenAndServe], +// and [ListenAndServeTLS] methods after a call to [Server.Shutdown] or [Server.Close]. +var http_ErrServerClosed = errors.New("http: Server closed") + // Serve accepts incoming connections on the Listener l, creating a -// new service goroutine for each. The service goroutines read requests and -// then call srv.Handler to reply to them. -func (srv *Server) Serve(l net.Listener) error { +// new service goroutine for each. The service goroutines read requests and +// then call s.Handler to reply to them. +// +// HTTP/2 support is only enabled if the Listener returns [*tls.Conn] +// connections and they were configured with "h2" in the TLS +// Config.NextProtos. +// +// Serve always returns a non-nil error and closes l. +// After [Server.Shutdown] or [Server.Close], the returned error is [ErrServerClosed]. +func (s *http_Server) Serve(l net.Listener) error { + if conf, ok := l.(http_http2ExternalServerConfig); ok { + // This is the sneaky path we use to let x/net/http2 wrap an http.Server: + // http2.ConfigureServer calls http.Server.Serve with a net.Listener that + // implements a certain interface, which we recognize here as an attempt + // to associate an http2.Server with us. + // + // (This is about as principled as the way we (ab)use Transport.RegisterProtocol, + // which is to say not at all. It's worth it.) + s.setHTTP2Config(conf) + // Server.Serve never returns a nil error under normal circumstances. + // Returning nil here informs our caller that we support this sneaky + // registration mechanism. + return nil + } + + if fn := http_testHookServerServe; fn != nil { + fn(s, l) // call hook with unwrapped listener + } + + origListener := l + l = &http_onceCloseListener{Listener: l} defer l.Close() + + if err := s.setupHTTP2_Serve(); err != nil { + return err + } + + if !s.trackListener(&l, true) { + return http_ErrServerClosed + } + defer s.trackListener(&l, false) + + baseCtx := context.Background() + if s.BaseContext != nil { + baseCtx = s.BaseContext(origListener) + if baseCtx == nil { + panic("BaseContext returned a nil context") + } + } + var tempDelay time.Duration // how long to sleep on accept failure + + ctx := context.WithValue(baseCtx, http_ServerContextKey, s) for { - rw, e := l.Accept() - if e != nil { - if ne, ok := e.(net.Error); ok && ne.Temporary() { + rw, err := l.Accept() + if err != nil { + if s.shuttingDown() { + return http_ErrServerClosed + } + if ne, ok := err.(net.Error); ok && ne.Temporary() { if tempDelay == 0 { tempDelay = 5 * time.Millisecond } else { @@ -4935,39 +11633,226 @@ if max := 1 * time.Second; tempDelay > max { tempDelay = max } - srv.logf("http: Accept error: %v; retrying in %v", e, tempDelay) + s.logf("http: Accept error: %v; retrying in %v", err, tempDelay) time.Sleep(tempDelay) continue } - return e + return err + } + connCtx := ctx + if cc := s.ConnContext; cc != nil { + connCtx = cc(connCtx, rw) + if connCtx == nil { + panic("ConnContext returned nil") + } } tempDelay = 0 - c, err := srv.newConn(rw) - if err != nil { - continue - } - c.setState(c.rwc, StateNew) - go c.serve() + c := s.newConn(rw) + c.setState(c.rwc, http_StateNew, http_runHooks) // before Serve can return + go c.serve(connCtx) } } -func (s *Server) doKeepAlives() bool { - return atomic.LoadInt32(&s.disableKeepAlives) == 0 +func (s *http_Server) setupTLSConfig(certFile, keyFile string, nextProtos []string) (*tls.Config, error) { + config := http_cloneTLSConfig(s.TLSConfig) + config.NextProtos = nextProtos + + configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil || config.GetConfigForClient != nil + if !configHasCert || certFile != "" || keyFile != "" { + var err error + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + } + return config, nil +} + +// ServeTLS accepts incoming connections on the Listener l, creating a +// new service goroutine for each. The service goroutines perform TLS +// setup and then read requests, calling s.Handler to reply to them. +// +// Files containing a certificate and matching private key for the +// server must be provided if neither the [Server]'s +// TLSConfig.Certificates, TLSConfig.GetCertificate nor +// config.GetConfigForClient are populated. +// If the certificate is signed by a certificate authority, the +// certFile should be the concatenation of the server's certificate, +// any intermediates, and the CA's certificate. +// +// ServeTLS always returns a non-nil error. After [Server.Shutdown] or [Server.Close], the +// returned error is [ErrServerClosed]. +func (s *http_Server) ServeTLS(l net.Listener, certFile, keyFile string) error { + // Setup HTTP/2 before s.Serve, to initialize s.TLSConfig + // before we clone it and create the TLS Listener. + if err := s.setupHTTP2_ServeTLS(); err != nil { + return err + } + + var nextProtos []string + if s.TLSConfig != nil { + nextProtos = s.TLSConfig.NextProtos + } + config, err := s.setupTLSConfig(certFile, keyFile, http_adjustNextProtos(nextProtos, s.protocols())) + if err != nil { + return err + } + + tlsListener := tls.NewListener(l, config) + return s.Serve(tlsListener) +} + +func (s *http_Server) protocols() http_Protocols { + if s.Protocols != nil { + // Historically, even when Protocols for a Server was set to be empty, + // the Server can still run normally with just HTTP/1. + // To keep backward-compatibility, the zero value of Protocols is + // defined as having only HTTP/1 enabled. + if s.Protocols.empty() { + var p http_Protocols + p.SetHTTP1(true) + return p + } + return *s.Protocols // user-configured set + } + + // The historic way of disabling HTTP/2 is to set TLSNextProto to + // a non-nil map with no "h2" entry. + _, hasH2 := s.TLSNextProto["h2"] + http2Disabled := s.TLSNextProto != nil && !hasH2 + + // If GODEBUG=http2server=0, then HTTP/2 is disabled unless + // the user has manually added an "h2" entry to TLSNextProto + // (probably by using x/net/http2 directly). + if http_http2server.Value() == "0" && !hasH2 { + http2Disabled = true + } + + var p http_Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + if !http2Disabled { + p.SetHTTP2(true) + } + return p +} + +// adjustNextProtos adds or removes "http/1.1" and "h2" entries from +// a tls.Config.NextProtos list, according to the set of protocols in protos. +func http_adjustNextProtos(nextProtos []string, protos http_Protocols) []string { + // Make a copy of NextProtos since it might be shared with some other tls.Config. + // (tls.Config.Clone doesn't do a deep copy.) + // + // We could avoid an allocation in the common case by checking to see if the slice + // is already in order, but this is just one small allocation per connection. + nextProtos = slices.Clone(nextProtos) + var have http_Protocols + nextProtos = slices.DeleteFunc(nextProtos, func(s string) bool { + switch s { + case "http/1.1": + if !protos.HTTP1() { + return true + } + have.SetHTTP1(true) + case "h2": + if !protos.HTTP2() { + return true + } + have.SetHTTP2(true) + } + return false + }) + if protos.HTTP2() && !have.HTTP2() { + nextProtos = append(nextProtos, "h2") + } + if protos.HTTP1() && !have.HTTP1() { + nextProtos = append(nextProtos, "http/1.1") + } + return nextProtos +} + +// trackListener adds or removes a net.Listener to the set of tracked +// listeners. +// +// We store a pointer to interface in the map set, in case the +// net.Listener is not comparable. This is safe because we only call +// trackListener via Serve and can track+defer untrack the same +// pointer to local variable there. We never need to compare a +// Listener from another caller. +// +// It reports whether the server is still up (not Shutdown or Closed). +func (s *http_Server) trackListener(ln *net.Listener, add bool) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.listeners == nil { + s.listeners = make(map[*net.Listener]struct{}) + } + if add { + if s.shuttingDown() { + return false + } + s.listeners[ln] = struct{}{} + s.listenerGroup.Add(1) + } else { + delete(s.listeners, ln) + s.listenerGroup.Done() + } + return true +} + +func (s *http_Server) trackConn(c *http_conn, add bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.activeConn == nil { + s.activeConn = make(map[*http_conn]struct{}) + } + if add { + s.activeConn[c] = struct{}{} + } else { + delete(s.activeConn, c) + } +} + +func (s *http_Server) idleTimeout() time.Duration { + if s.IdleTimeout != 0 { + return s.IdleTimeout + } + return s.ReadTimeout +} + +func (s *http_Server) readHeaderTimeout() time.Duration { + if s.ReadHeaderTimeout != 0 { + return s.ReadHeaderTimeout + } + return s.ReadTimeout +} + +func (s *http_Server) doKeepAlives() bool { + return !s.disableKeepAlives.Load() && !s.shuttingDown() +} + +func (s *http_Server) shuttingDown() bool { + return s.inShutdown.Load() } // SetKeepAlivesEnabled controls whether HTTP keep-alives are enabled. // By default, keep-alives are always enabled. Only very // resource-constrained environments or servers in the process of // shutting down should disable them. -func (srv *Server) SetKeepAlivesEnabled(v bool) { +func (s *http_Server) SetKeepAlivesEnabled(v bool) { if v { - atomic.StoreInt32(&srv.disableKeepAlives, 0) - } else { - atomic.StoreInt32(&srv.disableKeepAlives, 1) + s.disableKeepAlives.Store(false) + return } + s.disableKeepAlives.Store(true) + + // Close idle HTTP/1 conns: + s.closeIdleConns() + + // TODO: Issue 26303: close HTTP/2 conns as soon as they become idle. } -func (s *Server) logf(format string, args ...interface{}) { +func (s *http_Server) logf(format string, args ...any) { if s.ErrorLog != nil { s.ErrorLog.Printf(format, args...) } else { @@ -4975,258 +11860,413 @@ } } -// ListenAndServe listens on the TCP network address addr -// and then calls Serve with handler to handle requests -// on incoming connections. Handler is typically nil, -// in which case the DefaultServeMux is used. +// logf prints to the ErrorLog of the *Server associated with request r +// via ServerContextKey. If there's no associated server, or if ErrorLog +// is nil, logging is done via the log package's standard logger. +func http_logf(r *http_Request, format string, args ...any) { + s, _ := r.Context().Value(http_ServerContextKey).(*http_Server) + if s != nil && s.ErrorLog != nil { + s.ErrorLog.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +// ListenAndServe listens on the TCP network address addr and then calls +// [Serve] with handler to handle requests on incoming connections. +// Accepted connections are configured to enable TCP keep-alives. // -// A trivial example server is: +// The handler is typically nil, in which case [DefaultServeMux] is used. // -// package main -// -// import ( -// "io" -// "net/http" -// "log" -// ) -// -// // hello world, the web server -// func HelloServer(w http.ResponseWriter, req *http.Request) { -// io.WriteString(w, "hello, world!\n") -// } -// -// func main() { -// http.HandleFunc("/hello", HelloServer) -// err := http.ListenAndServe(":12345", nil) -// if err != nil { -// log.Fatal("ListenAndServe: ", err) -// } -// } -func ListenAndServe(addr string, handler Handler) error { - server := &Server{Addr: addr, Handler: handler} +// ListenAndServe always returns a non-nil error. +func http_ListenAndServe(addr string, handler http_Handler) error { + server := &http_Server{Addr: addr, Handler: handler} return server.ListenAndServe() } -// ListenAndServeTLS acts identically to ListenAndServe, except that it +// ListenAndServeTLS acts identically to [ListenAndServe], except that it // expects HTTPS connections. Additionally, files containing a certificate and // matching private key for the server must be provided. If the certificate // is signed by a certificate authority, the certFile should be the concatenation // of the server's certificate, any intermediates, and the CA's certificate. -// -// A trivial example server is: -// -// import ( -// "log" -// "net/http" -// ) -// -// func handler(w http.ResponseWriter, req *http.Request) { -// w.Header().Set("Content-Type", "text/plain") -// w.Write([]byte("This is an example server.\n")) -// } -// -// func main() { -// http.HandleFunc("/", handler) -// log.Printf("About to listen on 10443. Go to https://127.0.0.1:10443/") -// err := http.ListenAndServeTLS(":10443", "cert.pem", "key.pem", nil) -// if err != nil { -// log.Fatal(err) -// } -// } -// -// One can use generate_cert.go in crypto/tls to generate cert.pem and key.pem. -func ListenAndServeTLS(addr string, certFile string, keyFile string, handler Handler) error { - server := &Server{Addr: addr, Handler: handler} +func http_ListenAndServeTLS(addr, certFile, keyFile string, handler http_Handler) error { + server := &http_Server{Addr: addr, Handler: handler} return server.ListenAndServeTLS(certFile, keyFile) } -// ListenAndServeTLS listens on the TCP network address srv.Addr and -// then calls Serve to handle requests on incoming TLS connections. +// http3ServerHandler implements an interface in an external library that +// supports HTTP/3, allowing an external implementation of HTTP/3 to be used +// via net/http. See https://go.dev/issue/77440 for details. +// +// This is currently only used with golang.org/x/net/internal/http3, to allow +// us to test our HTTP/3 implementation against tests in net/http. HTTP/3 is +// not yet accessible to end-users. +type http_http3ServerHandler struct { + handler http_serverHandler + tlsConfig *tls.Config + baseCtx context.Context + errc chan error + shutdownCtx context.Context +} + +// ServeHTTP ensures that http3ServerHandler implements the Handler interface, +// and gives an HTTP/3 server implementation access to the net/http handler. +func (h *http_http3ServerHandler) ServeHTTP(w http_ResponseWriter, r *http_Request) { + h.handler.ServeHTTP(w, r) +} + +// Addr gives an HTTP/3 server implementation the address that it should listen +// on. +func (h *http_http3ServerHandler) Addr() string { + return h.handler.srv.Addr +} + +// TLSConfig gives an HTTP/3 server implementation the *tls.Config that it +// should use. +func (h *http_http3ServerHandler) TLSConfig() *tls.Config { + return h.tlsConfig +} + +// BaseContext gives an HTTP/3 server implementation the base context to use +// for server requests. +func (h *http_http3ServerHandler) BaseContext() context.Context { + return h.baseCtx +} + +// ListenErrHook should be called by an HTTP/3 server implementation to +// propagate any error it encounters when trying to listen, if any, to +// net/http. +func (h *http_http3ServerHandler) ListenErrHook(err error) { + h.errc <- err +} + +// ShutdownContext gives an HTTP/3 server implementation the context that is +// used when [Server.Shutdown] is called. This allows an HTTP/3 server +// implementation to know how long it can take to gracefully shutdown in the +// function it registers with [Server.RegisterOnShutdown]. Callers must not use +// this method for any other purpose. +func (h *http_http3ServerHandler) ShutdownContext() context.Context { + return h.shutdownCtx +} + +// ListenAndServeTLS listens on the TCP network address s.Addr and +// then calls [ServeTLS] to handle requests on incoming TLS connections. +// Accepted connections are configured to enable TCP keep-alives. // // Filenames containing a certificate and matching private key for the -// server must be provided if the Server's TLSConfig.Certificates is -// not populated. If the certificate is signed by a certificate -// authority, the certFile should be the concatenation of the server's -// certificate, any intermediates, and the CA's certificate. +// server must be provided if neither the [Server]'s TLSConfig.Certificates +// nor TLSConfig.GetCertificate are populated. If the certificate is +// signed by a certificate authority, the certFile should be the +// concatenation of the server's certificate, any intermediates, and +// the CA's certificate. // -// If srv.Addr is blank, ":https" is used. -func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { - addr := srv.Addr +// If s.Addr is blank, ":https" is used. +// +// ListenAndServeTLS always returns a non-nil error. After [Server.Shutdown] or +// [Server.Close], the returned error is [ErrServerClosed]. +func (s *http_Server) ListenAndServeTLS(certFile, keyFile string) error { + if s.shuttingDown() { + return http_ErrServerClosed + } + addr := s.Addr if addr == "" { addr = ":https" } - config := cloneTLSConfig(srv.TLSConfig) - if config.NextProtos == nil { - config.NextProtos = []string{"http/1.1"} - } - if len(config.Certificates) == 0 || certFile != "" || keyFile != "" { - var err error - config.Certificates = make([]tls.Certificate, 1) - config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + p := s.protocols() + if p.http3() { + fn, ok := s.TLSNextProto["http/3"] + if !ok { + return errors.New("http: Server.Protocols contains HTTP3, but Server does not support HTTP/3") + } + config, err := s.setupTLSConfig(certFile, keyFile, []string{"h3"}) if err != nil { return err } + errc := make(chan error, 1) + s.mu.Lock() + s.h3 = &http_http3ServerHandler{ + handler: http_serverHandler{s}, + tlsConfig: config, + baseCtx: context.WithValue(context.Background(), http_ServerContextKey, s), + errc: errc, + } + s.mu.Unlock() + go fn(s, nil, s.h3) + if err := <-errc; err != nil { + return err + } } + // Only start a TCP listener if HTTP/1 or HTTP/2 is used. + if !p.HTTP1() && !p.HTTP2() && !p.UnencryptedHTTP2() { + return nil + } ln, err := net.Listen("tcp", addr) if err != nil { return err } - - tlsListener := tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, config) - return srv.Serve(tlsListener) + defer ln.Close() + return s.ServeTLS(ln, certFile, keyFile) } -// TimeoutHandler returns a Handler that runs h with the given time limit. +// setupHTTP2_ServeTLS conditionally configures HTTP/2 on +// s and reports whether there was an error setting it up. If it is +// not configured for policy reasons, nil is returned. +func (s *http_Server) setupHTTP2_ServeTLS() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults) + return s.nextProtoErr +} + +// setupHTTP2_Serve is called from (*Server).Serve and conditionally +// configures HTTP/2 on s using a more conservative policy than +// setupHTTP2_ServeTLS because Serve is called after tls.Listen, +// and may be called concurrently. See shouldConfigureHTTP2ForServe. +// +// The tests named TestTransportAutomaticHTTP2* and +// TestConcurrentServerServe in server_test.go demonstrate some +// of the supported use cases and motivations. +func (s *http_Server) setupHTTP2_Serve() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults_Serve) + return s.nextProtoErr +} + +func (s *http_Server) onceSetNextProtoDefaults_Serve() { + if s.shouldConfigureHTTP2ForServe() { + s.onceSetNextProtoDefaults() + } +} + +var http_http2server = godebug.New("http2server") + +// onceSetNextProtoDefaults configures HTTP/2, if the user hasn't +// configured otherwise. (by setting s.TLSNextProto non-nil) +// It must only be called via s.nextProtoOnce (use s.setupHTTP2_*). +func (s *http_Server) onceSetNextProtoDefaults() { + if http_omitBundledHTTP2 { + return + } + p := s.protocols() + if !p.HTTP2() && !p.UnencryptedHTTP2() { + return + } + if http_http2server.Value() == "0" { + http_http2server.IncNonDefault() + return + } + if _, ok := s.TLSNextProto["h2"]; ok { + // TLSNextProto already contains an HTTP/2 implementation. + // The user probably called golang.org/x/net/http2.ConfigureServer + // to add it. + return + } + s.configureHTTP2() +} + +// TimeoutHandler returns a [Handler] that runs h with the given time limit. // // The new Handler calls h.ServeHTTP to handle each request, but if a // call runs for longer than its time limit, the handler responds with // a 503 Service Unavailable error and the given message in its body. // (If msg is empty, a suitable default message will be sent.) -// After such a timeout, writes by h to its ResponseWriter will return -// ErrHandlerTimeout. -func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { - f := func() <-chan time.Time { - return time.After(dt) +// After such a timeout, writes by h to its [ResponseWriter] will return +// [ErrHandlerTimeout]. +// +// TimeoutHandler supports the [Pusher] interface but does not support +// the [Hijacker] or [Flusher] interfaces. +func http_TimeoutHandler(h http_Handler, dt time.Duration, msg string) http_Handler { + return &http_timeoutHandler{ + handler: h, + body: msg, + dt: dt, } - return &timeoutHandler{h, f, msg} } -// ErrHandlerTimeout is returned on ResponseWriter Write calls +// ErrHandlerTimeout is returned on [ResponseWriter] Write calls // in handlers which have timed out. -var ErrHandlerTimeout = errors.New("http: Handler timeout") +var http_ErrHandlerTimeout = errors.New("http: Handler timeout") -type timeoutHandler struct { - handler Handler - timeout func() <-chan time.Time // returns channel producing a timeout +type http_timeoutHandler struct { + handler http_Handler body string + dt time.Duration + + // When set, no context will be created and this context will + // be used instead. + testContext context.Context } -func (h *timeoutHandler) errorBody() string { +func (h *http_timeoutHandler) errorBody() string { if h.body != "" { return h.body } return "<html><head><title>Timeout</title></head><body><h1>Timeout</h1></body></html>" } -func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { - done := make(chan bool, 1) - tw := &timeoutWriter{w: w} +func (h *http_timeoutHandler) ServeHTTP(w http_ResponseWriter, r *http_Request) { + ctx := h.testContext + if ctx == nil { + var cancelCtx context.CancelFunc + ctx, cancelCtx = context.WithTimeout(r.Context(), h.dt) + defer cancelCtx() + } + r = r.WithContext(ctx) + done := make(chan struct{}) + tw := &http_timeoutWriter{ + w: w, + h: make(http_Header), + req: r, + } + panicChan := make(chan any, 1) go func() { + defer func() { + if p := recover(); p != nil { + panicChan <- p + } + }() h.handler.ServeHTTP(tw, r) - done <- true + close(done) }() select { + case p := <-panicChan: + panic(p) case <-done: - return - case <-h.timeout(): tw.mu.Lock() defer tw.mu.Unlock() + dst := w.Header() + maps.Copy(dst, tw.h) if !tw.wroteHeader { - tw.w.WriteHeader(StatusServiceUnavailable) - tw.w.Write([]byte(h.errorBody())) + tw.code = http_StatusOK } - tw.timedOut = true + w.WriteHeader(tw.code) + w.Write(tw.wbuf.Bytes()) + case <-ctx.Done(): + tw.mu.Lock() + defer tw.mu.Unlock() + switch err := ctx.Err(); err { + case context.DeadlineExceeded: + w.WriteHeader(http_StatusServiceUnavailable) + io.WriteString(w, h.errorBody()) + tw.err = http_ErrHandlerTimeout + default: + w.WriteHeader(http_StatusServiceUnavailable) + tw.err = err + } } } -type timeoutWriter struct { - w ResponseWriter +type http_timeoutWriter struct { + w http_ResponseWriter + h http_Header + wbuf bytes.Buffer + req *http_Request mu sync.Mutex - timedOut bool + err error wroteHeader bool + code int } -func (tw *timeoutWriter) Header() Header { - return tw.w.Header() +var _ http_Pusher = (*http_timeoutWriter)(nil) + +// Push implements the [Pusher] interface. +func (tw *http_timeoutWriter) Push(target string, opts *http_PushOptions) error { + if pusher, ok := tw.w.(http_Pusher); ok { + return pusher.Push(target, opts) + } + return http_ErrNotSupported } -func (tw *timeoutWriter) Write(p []byte) (int, error) { +func (tw *http_timeoutWriter) Header() http_Header { return tw.h } + +func (tw *http_timeoutWriter) Write(p []byte) (int, error) { tw.mu.Lock() defer tw.mu.Unlock() - tw.wroteHeader = true - if tw.timedOut { - return 0, ErrHandlerTimeout + if tw.err != nil { + return 0, tw.err } - return tw.w.Write(p) + if !tw.wroteHeader { + tw.writeHeaderLocked(http_StatusOK) + } + return tw.wbuf.Write(p) } -func (tw *timeoutWriter) WriteHeader(code int) { +func (tw *http_timeoutWriter) writeHeaderLocked(code int) { + http_checkWriteHeaderCode(code) + + switch { + case tw.err != nil: + return + case tw.wroteHeader: + if tw.req != nil { + caller := http_relevantCaller() + http_logf(tw.req, "http: superfluous response.WriteHeader call from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + } + default: + tw.wroteHeader = true + tw.code = code + } +} + +func (tw *http_timeoutWriter) WriteHeader(code int) { tw.mu.Lock() defer tw.mu.Unlock() - if tw.timedOut || tw.wroteHeader { - return - } - tw.wroteHeader = true - tw.w.WriteHeader(code) + tw.writeHeaderLocked(code) } -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe and ListenAndServeTLS so -// dead TCP connections (e.g. closing laptop mid-download) eventually -// go away. -type tcpKeepAliveListener struct { - *net.TCPListener +// onceCloseListener wraps a net.Listener, protecting it from +// multiple Close calls. +type http_onceCloseListener struct { + net.Listener + once sync.Once + closeErr error } -func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { - tc, err := ln.AcceptTCP() - if err != nil { - return - } - tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(3 * time.Minute) - return tc, nil +func (oc *http_onceCloseListener) Close() error { + oc.once.Do(oc.close) + return oc.closeErr } +func (oc *http_onceCloseListener) close() { oc.closeErr = oc.Listener.Close() } + // globalOptionsHandler responds to "OPTIONS *" requests. -type globalOptionsHandler struct{} +type http_globalOptionsHandler struct{} -func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { +func (http_globalOptionsHandler) ServeHTTP(w http_ResponseWriter, r *http_Request) { w.Header().Set("Content-Length", "0") if r.ContentLength != 0 { - - mb := MaxBytesReader(w, r.Body, 4<<10) + // Read up to 4KB of OPTIONS body (as mentioned in the + // spec as being reserved for future use), but anything + // over that is considered a waste of server resources + // (or an attack) and we abort and close the connection, + // courtesy of MaxBytesReader's EOF behavior. + mb := http_MaxBytesReader(w, r.Body, 4<<10) io.Copy(io.Discard, mb) } } -type eofReaderWithWriteTo struct{} - -func (eofReaderWithWriteTo) WriteTo(io.Writer) (int64, error) { return 0, nil } - -func (eofReaderWithWriteTo) Read([]byte) (int, error) { return 0, io.EOF } - -// eofReader is a non-nil io.ReadCloser that always returns EOF. -// It has a WriteTo method so io.Copy won't need a buffer. -var eofReader = &struct { - eofReaderWithWriteTo - io.Closer -}{ - eofReaderWithWriteTo{}, - io.NopCloser(nil), -} - -// Verify that an io.Copy from an eofReader won't require a buffer. -var _ io.WriterTo = eofReader - -// initNPNRequest is an HTTP handler that initializes certain +// initALPNRequest is an HTTP handler that initializes certain // uninitialized fields in its *Request. Such partially-initialized -// Requests come from NPN protocol handlers. -type initNPNRequest struct { - c *tls.Conn - h serverHandler +// Requests come from ALPN protocol handlers. +type http_initALPNRequest struct { + ctx context.Context + c *tls.Conn + h http_serverHandler } -func (h initNPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { +// BaseContext is an exported but unadvertised [http.Handler] method +// recognized by x/net/http2 to pass down a context; the TLSNextProto +// API predates context support so we shoehorn through the only +// interface we have available. +func (h http_initALPNRequest) BaseContext() context.Context { return h.ctx } + +func (h http_initALPNRequest) ServeHTTP(rw http_ResponseWriter, req *http_Request) { if req.TLS == nil { req.TLS = &tls.ConnectionState{} *req.TLS = h.c.ConnectionState() } if req.Body == nil { - req.Body = eofReader + req.Body = http_NoBody } if req.RemoteAddr == "" { req.RemoteAddr = h.c.RemoteAddr().String() @@ -5235,41 +12275,41 @@ } // loggingConn is used for debugging. -type loggingConn struct { +type http_loggingConn struct { name string net.Conn } var ( - uniqNameMu sync.Mutex - uniqNameNext = make(map[string]int) + http_uniqNameMu sync.Mutex + http_uniqNameNext = make(map[string]int) ) -func newLoggingConn(baseName string, c net.Conn) net.Conn { - uniqNameMu.Lock() - defer uniqNameMu.Unlock() - uniqNameNext[baseName]++ - return &loggingConn{ - name: fmt.Sprintf("%s-%d", baseName, uniqNameNext[baseName]), +func http_newLoggingConn(baseName string, c net.Conn) net.Conn { + http_uniqNameMu.Lock() + defer http_uniqNameMu.Unlock() + http_uniqNameNext[baseName]++ + return &http_loggingConn{ + name: fmt.Sprintf("%s-%d", baseName, http_uniqNameNext[baseName]), Conn: c, } } -func (c *loggingConn) Write(p []byte) (n int, err error) { +func (c *http_loggingConn) Write(p []byte) (n int, err error) { log.Printf("%s.Write(%d) = ....", c.name, len(p)) n, err = c.Conn.Write(p) log.Printf("%s.Write(%d) = %d, %v", c.name, len(p), n, err) return } -func (c *loggingConn) Read(p []byte) (n int, err error) { +func (c *http_loggingConn) Read(p []byte) (n int, err error) { log.Printf("%s.Read(%d) = ....", c.name, len(p)) n, err = c.Conn.Read(p) log.Printf("%s.Read(%d) = %d, %v", c.name, len(p), n, err) return } -func (c *loggingConn) Close() (err error) { +func (c *http_loggingConn) Close() (err error) { log.Printf("%s.Close() = ...", c.name) err = c.Conn.Close() log.Printf("%s.Close() = %v", c.name, err) @@ -5279,19 +12319,20 @@ // checkConnErrorWriter writes to c.rwc and records any write errors to c.werr. // It only contains one field (and a pointer field at that), so it // fits in an interface value without an extra allocation. -type checkConnErrorWriter struct { - c *conn +type http_checkConnErrorWriter struct { + c *http_conn } -func (w checkConnErrorWriter) Write(p []byte) (n int, err error) { - n, err = w.c.w.Write(p) +func (w http_checkConnErrorWriter) Write(p []byte) (n int, err error) { + n, err = w.c.rwc.Write(p) if err != nil && w.c.werr == nil { w.c.werr = err + w.c.cancelCtx() } return } -func numLeadingCRorLF(v []byte) (n int) { +func http_numLeadingCRorLF(v []byte) (n int) { for _, b := range v { if b == '\r' || b == '\n' { n++ @@ -5300,338 +12341,728 @@ break } return - } -// The algorithm uses at most sniffLen bytes to make its decision. -const sniffLen = 512 - -// DetectContentType implements the algorithm described -// at http://mimesniff.spec.whatwg.org/ to determine the -// Content-Type of the given data. It considers at most the -// first 512 bytes of data. DetectContentType always returns -// a valid MIME type: if it cannot determine a more specific one, it -// returns "application/octet-stream". -func DetectContentType(data []byte) string { - if len(data) > sniffLen { - data = data[:sniffLen] - } - - firstNonWS := 0 - for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ { - } - - for _, sig := range sniffSignatures { - if ct := sig.match(data, firstNonWS); ct != "" { - return ct - } - } - - return "application/octet-stream" -} - -func isWS(b byte) bool { - switch b { - case '\t', '\n', '\x0c', '\r', ' ': +// tlsRecordHeaderLooksLikeHTTP reports whether a TLS record header +// looks like it might've been a misdirected plaintext HTTP request. +func http_tlsRecordHeaderLooksLikeHTTP(hdr [5]byte) bool { + switch string(hdr[:]) { + case "GET /", "HEAD ", "POST ", "PUT /", "OPTIO": return true } return false } -type sniffSig interface { - // match returns the MIME type of the data, or "" if unknown. - match(data []byte, firstNonWS int) string +// MaxBytesHandler returns a [Handler] that runs h with its [ResponseWriter] and [Request.Body] wrapped by a MaxBytesReader. +func http_MaxBytesHandler(h http_Handler, n int64) http_Handler { + return http_HandlerFunc(func(w http_ResponseWriter, r *http_Request) { + r2 := *r + r2.Body = http_MaxBytesReader(w, r.Body, n) + h.ServeHTTP(w, &r2) + }) } -// Data matching the table in section 6. -var sniffSignatures = []sniffSig{ - htmlSig("<!DOCTYPE HTML"), - htmlSig("<HTML"), - htmlSig("<HEAD"), - htmlSig("<SCRIPT"), - htmlSig("<IFRAME"), - htmlSig("<H1"), - htmlSig("<DIV"), - htmlSig("<FONT"), - htmlSig("<TABLE"), - htmlSig("<A"), - htmlSig("<STYLE"), - htmlSig("<TITLE"), - htmlSig("<B"), - htmlSig("<BODY"), - htmlSig("<BR"), - htmlSig("<P"), - htmlSig("<!--"), - - &maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"}, - - &exactSig{[]byte("%PDF-"), "application/pdf"}, - &exactSig{[]byte("%!PS-Adobe-"), "application/postscript"}, - - &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"}, - &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"}, - &maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"}, - - &exactSig{[]byte("GIF87a"), "image/gif"}, - &exactSig{[]byte("GIF89a"), "image/gif"}, - &exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"}, - &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"}, - &exactSig{[]byte("BM"), "image/bmp"}, - &maskedSig{ - mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"), - pat: []byte("RIFF\x00\x00\x00\x00WEBPVP"), - ct: "image/webp", - }, - &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"}, - &exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"}, - &maskedSig{ - mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), - pat: []byte("RIFF\x00\x00\x00\x00WAVE"), - ct: "audio/wave", - }, - &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, - &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"}, - &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"}, - &exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"}, - - textSig(0), +// DetectContentType implements the algorithm described +// at https://mimesniff.spec.whatwg.org/ to determine the +// Content-Type of the given data. It considers at most the +// first 512 bytes of data. DetectContentType always returns +// a valid MIME type: if it cannot determine a more specific one, it +// returns "application/octet-stream". +func http_DetectContentType(data []byte) string { + return internal.DetectContentType(data) } -type exactSig struct { - sig []byte - ct string -} - -func (e *exactSig) match(data []byte, firstNonWS int) string { - if bytes.HasPrefix(data, e.sig) { - return e.ct - } - return "" -} - -type maskedSig struct { - mask, pat []byte - skipWS bool - ct string -} - -func (m *maskedSig) match(data []byte, firstNonWS int) string { - if m.skipWS { - data = data[firstNonWS:] - } - if len(data) < len(m.mask) { - return "" - } - for i, mask := range m.mask { - db := data[i] & mask - if db != m.pat[i] { - return "" - } - } - return m.ct -} - -type htmlSig []byte - -func (h htmlSig) match(data []byte, firstNonWS int) string { - data = data[firstNonWS:] - if len(data) < len(h)+1 { - return "" - } - for i, b := range h { - db := data[i] - if 'A' <= b && b <= 'Z' { - db &= 0xDF - } - if b != db { - return "" - } - } - - if db := data[len(h)]; db != ' ' && db != '>' { - return "" - } - return "text/html; charset=utf-8" -} - -var mp4ftype = []byte("ftyp") - -type mp4Sig int - -func (mp4Sig) match(data []byte, firstNonWS int) string { - - if len(data) < 8 { - return "" - } - boxSize := int(binary.BigEndian.Uint32(data[:4])) - if boxSize%4 != 0 || len(data) < boxSize { - return "" - } - if !bytes.Equal(data[4:8], mp4ftype) { - return "" - } - for st := 8; st < boxSize; st += 4 { - if st == 12 { - - continue - } - seg := string(data[st : st+3]) - switch seg { - case "mp4", "iso", "M4V", "M4P", "M4B": - return "video/mp4" - - } - } - return "" -} - -type textSig int - -func (textSig) match(data []byte, firstNonWS int) string { - - for _, b := range data[firstNonWS:] { - switch { - case 0x00 <= b && b <= 0x08, - b == 0x0B, - 0x0E <= b && b <= 0x1A, - 0x1C <= b && b <= 0x1F: - return "" - } - } - return "text/plain; charset=utf-8" -} - -// HTTP status codes, defined in RFC 2616. -const ( - StatusContinue = 100 - StatusSwitchingProtocols = 101 - - StatusOK = 200 - StatusCreated = 201 - StatusAccepted = 202 - StatusNonAuthoritativeInfo = 203 - StatusNoContent = 204 - StatusResetContent = 205 - StatusPartialContent = 206 - - StatusMultipleChoices = 300 - StatusMovedPermanently = 301 - StatusFound = 302 - StatusSeeOther = 303 - StatusNotModified = 304 - StatusUseProxy = 305 - StatusTemporaryRedirect = 307 - - StatusBadRequest = 400 - StatusUnauthorized = 401 - StatusPaymentRequired = 402 - StatusForbidden = 403 - StatusNotFound = 404 - StatusMethodNotAllowed = 405 - StatusNotAcceptable = 406 - StatusProxyAuthRequired = 407 - StatusRequestTimeout = 408 - StatusConflict = 409 - StatusGone = 410 - StatusLengthRequired = 411 - StatusPreconditionFailed = 412 - StatusRequestEntityTooLarge = 413 - StatusRequestURITooLong = 414 - StatusUnsupportedMediaType = 415 - StatusRequestedRangeNotSatisfiable = 416 - StatusExpectationFailed = 417 - StatusTeapot = 418 - - StatusInternalServerError = 500 - StatusNotImplemented = 501 - StatusBadGateway = 502 - StatusServiceUnavailable = 503 - StatusGatewayTimeout = 504 - StatusHTTPVersionNotSupported = 505 - - // New HTTP status codes from RFC 6585. Not exported yet in Go 1.1. - // See discussion at https://codereview.appspot.com/7678043/ - statusPreconditionRequired = 428 - statusTooManyRequests = 429 - statusRequestHeaderFieldsTooLarge = 431 - statusNetworkAuthenticationRequired = 511 +var ( + http_socksnoDeadline = time.Time{} + http_socksaLongTimeAgo = time.Unix(1, 0) ) -var statusText = map[int]string{ - StatusContinue: "Continue", - StatusSwitchingProtocols: "Switching Protocols", +func (d *http_socksDialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) { + host, port, err := http_sockssplitHostPort(address) + if err != nil { + return nil, err + } + if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { + c.SetDeadline(deadline) + defer c.SetDeadline(http_socksnoDeadline) + } + if ctx != context.Background() { + errCh := make(chan error, 1) + done := make(chan struct{}) + defer func() { + close(done) + if ctxErr == nil { + ctxErr = <-errCh + } + }() + go func() { + select { + case <-ctx.Done(): + c.SetDeadline(http_socksaLongTimeAgo) + errCh <- ctx.Err() + case <-done: + errCh <- nil + } + }() + } - StatusOK: "OK", - StatusCreated: "Created", - StatusAccepted: "Accepted", - StatusNonAuthoritativeInfo: "Non-Authoritative Information", - StatusNoContent: "No Content", - StatusResetContent: "Reset Content", - StatusPartialContent: "Partial Content", + b := make([]byte, 0, 6+len(host)) // the size here is just an estimate + b = append(b, http_socksVersion5) + if len(d.AuthMethods) == 0 || d.Authenticate == nil { + b = append(b, 1, byte(http_socksAuthMethodNotRequired)) + } else { + ams := d.AuthMethods + if len(ams) > 255 { + return nil, errors.New("too many authentication methods") + } + b = append(b, byte(len(ams))) + for _, am := range ams { + b = append(b, byte(am)) + } + } + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } - StatusMultipleChoices: "Multiple Choices", - StatusMovedPermanently: "Moved Permanently", - StatusFound: "Found", - StatusSeeOther: "See Other", - StatusNotModified: "Not Modified", - StatusUseProxy: "Use Proxy", - StatusTemporaryRedirect: "Temporary Redirect", + if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil { + return + } + if b[0] != http_socksVersion5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + am := http_socksAuthMethod(b[1]) + if am == http_socksAuthMethodNoAcceptableMethods { + return nil, errors.New("no acceptable authentication methods") + } + if d.Authenticate != nil { + if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil { + return + } + } - StatusBadRequest: "Bad Request", - StatusUnauthorized: "Unauthorized", - StatusPaymentRequired: "Payment Required", - StatusForbidden: "Forbidden", - StatusNotFound: "Not Found", - StatusMethodNotAllowed: "Method Not Allowed", - StatusNotAcceptable: "Not Acceptable", - StatusProxyAuthRequired: "Proxy Authentication Required", - StatusRequestTimeout: "Request Timeout", - StatusConflict: "Conflict", - StatusGone: "Gone", - StatusLengthRequired: "Length Required", - StatusPreconditionFailed: "Precondition Failed", - StatusRequestEntityTooLarge: "Request Entity Too Large", - StatusRequestURITooLong: "Request URI Too Long", - StatusUnsupportedMediaType: "Unsupported Media Type", - StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", - StatusExpectationFailed: "Expectation Failed", - StatusTeapot: "I'm a teapot", + b = b[:0] + b = append(b, http_socksVersion5, byte(d.cmd), 0) + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + b = append(b, http_socksAddrTypeIPv4) + b = append(b, ip4...) + } else if ip6 := ip.To16(); ip6 != nil { + b = append(b, http_socksAddrTypeIPv6) + b = append(b, ip6...) + } else { + return nil, errors.New("unknown address type") + } + } else { + if len(host) > 255 { + return nil, errors.New("FQDN too long") + } + b = append(b, http_socksAddrTypeFQDN) + b = append(b, byte(len(host))) + b = append(b, host...) + } + b = append(b, byte(port>>8), byte(port)) + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } - StatusInternalServerError: "Internal Server Error", - StatusNotImplemented: "Not Implemented", - StatusBadGateway: "Bad Gateway", - StatusServiceUnavailable: "Service Unavailable", - StatusGatewayTimeout: "Gateway Timeout", - StatusHTTPVersionNotSupported: "HTTP Version Not Supported", - - statusPreconditionRequired: "Precondition Required", - statusTooManyRequests: "Too Many Requests", - statusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", - statusNetworkAuthenticationRequired: "Network Authentication Required", + if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil { + return + } + if b[0] != http_socksVersion5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + if cmdErr := http_socksReply(b[1]); cmdErr != http_socksStatusSucceeded { + return nil, errors.New("unknown error " + cmdErr.String()) + } + if b[2] != 0 { + return nil, errors.New("non-zero reserved field") + } + l := 2 + var a http_socksAddr + switch b[3] { + case http_socksAddrTypeIPv4: + l += net.IPv4len + a.IP = make(net.IP, net.IPv4len) + case http_socksAddrTypeIPv6: + l += net.IPv6len + a.IP = make(net.IP, net.IPv6len) + case http_socksAddrTypeFQDN: + if _, err := io.ReadFull(c, b[:1]); err != nil { + return nil, err + } + l += int(b[0]) + default: + return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3]))) + } + if cap(b) < l { + b = make([]byte, l) + } else { + b = b[:l] + } + if _, ctxErr = io.ReadFull(c, b); ctxErr != nil { + return + } + if a.IP != nil { + copy(a.IP, b) + } else { + a.Name = string(b[:len(b)-2]) + } + a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1]) + return &a, nil } +func http_sockssplitHostPort(address string) (string, int, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return "", 0, err + } + portnum, err := strconv.Atoi(port) + if err != nil { + return "", 0, err + } + if 1 > portnum || portnum > 0xffff { + return "", 0, errors.New("port number out of range " + port) + } + return host, portnum, nil +} + +// A Command represents a SOCKS command. +type http_socksCommand int + +func (cmd http_socksCommand) String() string { + switch cmd { + case http_socksCmdConnect: + return "socks connect" + case http_sockscmdBind: + return "socks bind" + default: + return "socks " + strconv.Itoa(int(cmd)) + } +} + +// An AuthMethod represents a SOCKS authentication method. +type http_socksAuthMethod int + +// A Reply represents a SOCKS command reply code. +type http_socksReply int + +func (code http_socksReply) String() string { + switch code { + case http_socksStatusSucceeded: + return "succeeded" + case 0x01: + return "general SOCKS server failure" + case 0x02: + return "connection not allowed by ruleset" + case 0x03: + return "network unreachable" + case 0x04: + return "host unreachable" + case 0x05: + return "connection refused" + case 0x06: + return "TTL expired" + case 0x07: + return "command not supported" + case 0x08: + return "address type not supported" + default: + return "unknown code: " + strconv.Itoa(int(code)) + } +} + +// Wire protocol constants. +const ( + http_socksVersion5 = 0x05 + + http_socksAddrTypeIPv4 = 0x01 + http_socksAddrTypeFQDN = 0x03 + http_socksAddrTypeIPv6 = 0x04 + + http_socksCmdConnect http_socksCommand = 0x01 // establishes an active-open forward proxy connection + http_sockscmdBind http_socksCommand = 0x02 // establishes a passive-open forward proxy connection + + http_socksAuthMethodNotRequired http_socksAuthMethod = 0x00 // no authentication required + http_socksAuthMethodUsernamePassword http_socksAuthMethod = 0x02 // use username/password + http_socksAuthMethodNoAcceptableMethods http_socksAuthMethod = 0xff // no acceptable authentication methods + + http_socksStatusSucceeded http_socksReply = 0x00 +) + +// An Addr represents a SOCKS-specific address. +// Either Name or IP is used exclusively. +type http_socksAddr struct { + Name string // fully-qualified domain name + IP net.IP + Port int +} + +func (a *http_socksAddr) Network() string { return "socks" } + +func (a *http_socksAddr) String() string { + if a == nil { + return "<nil>" + } + port := strconv.Itoa(a.Port) + if a.IP == nil { + return net.JoinHostPort(a.Name, port) + } + return net.JoinHostPort(a.IP.String(), port) +} + +// A Conn represents a forward proxy connection. +type http_socksConn struct { + net.Conn + + boundAddr net.Addr +} + +// BoundAddr returns the address assigned by the proxy server for +// connecting to the command target address from the proxy server. +func (c *http_socksConn) BoundAddr() net.Addr { + if c == nil { + return nil + } + return c.boundAddr +} + +// A Dialer holds SOCKS-specific options. +type http_socksDialer struct { + cmd http_socksCommand // either CmdConnect or cmdBind + proxyNetwork string // network between a proxy server and a client + proxyAddress string // proxy server address + + // ProxyDial specifies the optional dial function for + // establishing the transport connection. + ProxyDial func(context.Context, string, string) (net.Conn, error) + + // AuthMethods specifies the list of request authentication + // methods. + // If empty, SOCKS client requests only AuthMethodNotRequired. + AuthMethods []http_socksAuthMethod + + // Authenticate specifies the optional authentication + // function. It must be non-nil when AuthMethods is not empty. + // It must return an error when the authentication is failed. + Authenticate func(context.Context, io.ReadWriter, http_socksAuthMethod) error +} + +// DialContext connects to the provided address on the provided +// network. +// +// The returned error value may be a net.OpError. When the Op field of +// net.OpError contains "socks", the Source field contains a proxy +// server address and the Addr field contains a command target +// address. +// +// See func Dial of the net package of standard library for a +// description of the network and address parameters. +func (d *http_socksDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress) + } else { + var dd net.Dialer + c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + a, err := d.connect(ctx, c, address) + if err != nil { + c.Close() + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return &http_socksConn{Conn: c, boundAddr: a}, nil +} + +// DialWithConn initiates a connection from SOCKS server to the target +// network and address using the connection c that is already +// connected to the SOCKS server. +// +// It returns the connection's local address assigned by the SOCKS +// server. +func (d *http_socksDialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + a, err := d.connect(ctx, c, address) + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return a, nil +} + +// Dial connects to the provided address on the provided network. +// +// Unlike DialContext, it returns a raw transport connection instead +// of a forward proxy connection. +// +// Deprecated: Use DialContext or DialWithConn instead. +func (d *http_socksDialer) Dial(network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) + } else { + c, err = net.Dial(d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { + c.Close() + return nil, err + } + return c, nil +} + +func (d *http_socksDialer) validateTarget(network, address string) error { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return errors.New("network not implemented") + } + switch d.cmd { + case http_socksCmdConnect, http_sockscmdBind: + default: + return errors.New("command not implemented") + } + return nil +} + +func (d *http_socksDialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { + for i, s := range []string{d.proxyAddress, address} { + host, port, err := http_sockssplitHostPort(s) + if err != nil { + return nil, nil, err + } + a := &http_socksAddr{Port: port} + a.IP = net.ParseIP(host) + if a.IP == nil { + a.Name = host + } + if i == 0 { + proxy = a + } else { + dst = a + } + } + return +} + +// NewDialer returns a new Dialer that dials through the provided +// proxy server's network and address. +func http_socksNewDialer(network, address string) *http_socksDialer { + return &http_socksDialer{proxyNetwork: network, proxyAddress: address, cmd: http_socksCmdConnect} +} + +const ( + http_socksauthUsernamePasswordVersion = 0x01 + http_socksauthStatusSucceeded = 0x00 +) + +// UsernamePassword are the credentials for the username/password +// authentication method. +type http_socksUsernamePassword struct { + Username string + Password string +} + +// Authenticate authenticates a pair of username and password with the +// proxy server. +func (up *http_socksUsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth http_socksAuthMethod) error { + switch auth { + case http_socksAuthMethodNotRequired: + return nil + case http_socksAuthMethodUsernamePassword: + if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 { + return errors.New("invalid username/password") + } + b := []byte{http_socksauthUsernamePasswordVersion} + b = append(b, byte(len(up.Username))) + b = append(b, up.Username...) + b = append(b, byte(len(up.Password))) + b = append(b, up.Password...) + // TODO(mikio): handle IO deadlines and cancellation if + // necessary + if _, err := rw.Write(b); err != nil { + return err + } + if _, err := io.ReadFull(rw, b[:2]); err != nil { + return err + } + if b[0] != http_socksauthUsernamePasswordVersion { + return errors.New("invalid username/password version") + } + if b[1] != http_socksauthStatusSucceeded { + return errors.New("username/password authentication failed") + } + return nil + } + return errors.New("unsupported authentication method " + strconv.Itoa(int(auth))) +} + +// HTTP status codes as registered with IANA. +// See: https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml +const ( + http_StatusContinue = 100 // RFC 9110, 15.2.1 + http_StatusSwitchingProtocols = 101 // RFC 9110, 15.2.2 + http_StatusProcessing = 102 // RFC 2518, 10.1 + http_StatusEarlyHints = 103 // RFC 8297 + + http_StatusOK = 200 // RFC 9110, 15.3.1 + http_StatusCreated = 201 // RFC 9110, 15.3.2 + http_StatusAccepted = 202 // RFC 9110, 15.3.3 + http_StatusNonAuthoritativeInfo = 203 // RFC 9110, 15.3.4 + http_StatusNoContent = 204 // RFC 9110, 15.3.5 + http_StatusResetContent = 205 // RFC 9110, 15.3.6 + http_StatusPartialContent = 206 // RFC 9110, 15.3.7 + http_StatusMultiStatus = 207 // RFC 4918, 11.1 + http_StatusAlreadyReported = 208 // RFC 5842, 7.1 + http_StatusIMUsed = 226 // RFC 3229, 10.4.1 + + http_StatusMultipleChoices = 300 // RFC 9110, 15.4.1 + http_StatusMovedPermanently = 301 // RFC 9110, 15.4.2 + http_StatusFound = 302 // RFC 9110, 15.4.3 + http_StatusSeeOther = 303 // RFC 9110, 15.4.4 + http_StatusNotModified = 304 // RFC 9110, 15.4.5 + http_StatusUseProxy = 305 // RFC 9110, 15.4.6 + _ = 306 // RFC 9110, 15.4.7 (Unused) + http_StatusTemporaryRedirect = 307 // RFC 9110, 15.4.8 + http_StatusPermanentRedirect = 308 // RFC 9110, 15.4.9 + + http_StatusBadRequest = 400 // RFC 9110, 15.5.1 + http_StatusUnauthorized = 401 // RFC 9110, 15.5.2 + http_StatusPaymentRequired = 402 // RFC 9110, 15.5.3 + http_StatusForbidden = 403 // RFC 9110, 15.5.4 + http_StatusNotFound = 404 // RFC 9110, 15.5.5 + http_StatusMethodNotAllowed = 405 // RFC 9110, 15.5.6 + http_StatusNotAcceptable = 406 // RFC 9110, 15.5.7 + http_StatusProxyAuthRequired = 407 // RFC 9110, 15.5.8 + http_StatusRequestTimeout = 408 // RFC 9110, 15.5.9 + http_StatusConflict = 409 // RFC 9110, 15.5.10 + http_StatusGone = 410 // RFC 9110, 15.5.11 + http_StatusLengthRequired = 411 // RFC 9110, 15.5.12 + http_StatusPreconditionFailed = 412 // RFC 9110, 15.5.13 + http_StatusRequestEntityTooLarge = 413 // RFC 9110, 15.5.14 + http_StatusRequestURITooLong = 414 // RFC 9110, 15.5.15 + http_StatusUnsupportedMediaType = 415 // RFC 9110, 15.5.16 + http_StatusRequestedRangeNotSatisfiable = 416 // RFC 9110, 15.5.17 + http_StatusExpectationFailed = 417 // RFC 9110, 15.5.18 + http_StatusTeapot = 418 // RFC 9110, 15.5.19 (Unused) + http_StatusMisdirectedRequest = 421 // RFC 9110, 15.5.20 + http_StatusUnprocessableEntity = 422 // RFC 9110, 15.5.21 + http_StatusLocked = 423 // RFC 4918, 11.3 + http_StatusFailedDependency = 424 // RFC 4918, 11.4 + http_StatusTooEarly = 425 // RFC 8470, 5.2. + http_StatusUpgradeRequired = 426 // RFC 9110, 15.5.22 + http_StatusPreconditionRequired = 428 // RFC 6585, 3 + http_StatusTooManyRequests = 429 // RFC 6585, 4 + http_StatusRequestHeaderFieldsTooLarge = 431 // RFC 6585, 5 + http_StatusUnavailableForLegalReasons = 451 // RFC 7725, 3 + + http_StatusInternalServerError = 500 // RFC 9110, 15.6.1 + http_StatusNotImplemented = 501 // RFC 9110, 15.6.2 + http_StatusBadGateway = 502 // RFC 9110, 15.6.3 + http_StatusServiceUnavailable = 503 // RFC 9110, 15.6.4 + http_StatusGatewayTimeout = 504 // RFC 9110, 15.6.5 + http_StatusHTTPVersionNotSupported = 505 // RFC 9110, 15.6.6 + http_StatusVariantAlsoNegotiates = 506 // RFC 2295, 8.1 + http_StatusInsufficientStorage = 507 // RFC 4918, 11.5 + http_StatusLoopDetected = 508 // RFC 5842, 7.2 + http_StatusNotExtended = 510 // RFC 2774, 7 + http_StatusNetworkAuthenticationRequired = 511 // RFC 6585, 6 +) + // StatusText returns a text for the HTTP status code. It returns the empty // string if the code is unknown. -func StatusText(code int) string { - return statusText[code] +func http_StatusText(code int) string { + switch code { + case http_StatusContinue: + return "Continue" + case http_StatusSwitchingProtocols: + return "Switching Protocols" + case http_StatusProcessing: + return "Processing" + case http_StatusEarlyHints: + return "Early Hints" + case http_StatusOK: + return "OK" + case http_StatusCreated: + return "Created" + case http_StatusAccepted: + return "Accepted" + case http_StatusNonAuthoritativeInfo: + return "Non-Authoritative Information" + case http_StatusNoContent: + return "No Content" + case http_StatusResetContent: + return "Reset Content" + case http_StatusPartialContent: + return "Partial Content" + case http_StatusMultiStatus: + return "Multi-Status" + case http_StatusAlreadyReported: + return "Already Reported" + case http_StatusIMUsed: + return "IM Used" + case http_StatusMultipleChoices: + return "Multiple Choices" + case http_StatusMovedPermanently: + return "Moved Permanently" + case http_StatusFound: + return "Found" + case http_StatusSeeOther: + return "See Other" + case http_StatusNotModified: + return "Not Modified" + case http_StatusUseProxy: + return "Use Proxy" + case http_StatusTemporaryRedirect: + return "Temporary Redirect" + case http_StatusPermanentRedirect: + return "Permanent Redirect" + case http_StatusBadRequest: + return "Bad Request" + case http_StatusUnauthorized: + return "Unauthorized" + case http_StatusPaymentRequired: + return "Payment Required" + case http_StatusForbidden: + return "Forbidden" + case http_StatusNotFound: + return "Not Found" + case http_StatusMethodNotAllowed: + return "Method Not Allowed" + case http_StatusNotAcceptable: + return "Not Acceptable" + case http_StatusProxyAuthRequired: + return "Proxy Authentication Required" + case http_StatusRequestTimeout: + return "Request Timeout" + case http_StatusConflict: + return "Conflict" + case http_StatusGone: + return "Gone" + case http_StatusLengthRequired: + return "Length Required" + case http_StatusPreconditionFailed: + return "Precondition Failed" + case http_StatusRequestEntityTooLarge: + return "Request Entity Too Large" + case http_StatusRequestURITooLong: + return "Request URI Too Long" + case http_StatusUnsupportedMediaType: + return "Unsupported Media Type" + case http_StatusRequestedRangeNotSatisfiable: + return "Requested Range Not Satisfiable" + case http_StatusExpectationFailed: + return "Expectation Failed" + case http_StatusTeapot: + return "I'm a teapot" + case http_StatusMisdirectedRequest: + return "Misdirected Request" + case http_StatusUnprocessableEntity: + return "Unprocessable Entity" + case http_StatusLocked: + return "Locked" + case http_StatusFailedDependency: + return "Failed Dependency" + case http_StatusTooEarly: + return "Too Early" + case http_StatusUpgradeRequired: + return "Upgrade Required" + case http_StatusPreconditionRequired: + return "Precondition Required" + case http_StatusTooManyRequests: + return "Too Many Requests" + case http_StatusRequestHeaderFieldsTooLarge: + return "Request Header Fields Too Large" + case http_StatusUnavailableForLegalReasons: + return "Unavailable For Legal Reasons" + case http_StatusInternalServerError: + return "Internal Server Error" + case http_StatusNotImplemented: + return "Not Implemented" + case http_StatusBadGateway: + return "Bad Gateway" + case http_StatusServiceUnavailable: + return "Service Unavailable" + case http_StatusGatewayTimeout: + return "Gateway Timeout" + case http_StatusHTTPVersionNotSupported: + return "HTTP Version Not Supported" + case http_StatusVariantAlsoNegotiates: + return "Variant Also Negotiates" + case http_StatusInsufficientStorage: + return "Insufficient Storage" + case http_StatusLoopDetected: + return "Loop Detected" + case http_StatusNotExtended: + return "Not Extended" + case http_StatusNetworkAuthenticationRequired: + return "Network Authentication Required" + default: + return "" + } } // ErrLineTooLong is returned when reading request or response bodies // with malformed chunked encoding. -var ErrLineTooLong = internal.ErrLineTooLong +var http_ErrLineTooLong = internal.ErrLineTooLong -type errorReader struct { +type http_errorReader struct { err error } -func (r errorReader) Read(p []byte) (n int, err error) { +func (r http_errorReader) Read(p []byte) (n int, err error) { return 0, r.err } +type http_byteReader struct { + b byte + done bool +} + +func (br *http_byteReader) Read(p []byte) (n int, err error) { + if br.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + br.done = true + p[0] = br.b + return 1, io.EOF +} + // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. -type transferWriter struct { +type http_transferWriter struct { Method string Body io.Reader BodyCloser io.Closer @@ -5639,50 +13070,49 @@ ContentLength int64 // -1 means unknown, 0 means exactly none Close bool TransferEncoding []string - Trailer Header + Header http_Header + Trailer http_Header IsResponse bool + bodyReadError error // any non-EOF error from reading Body + + FlushHeaders bool // flush headers to network before body + ByteReadCh chan http_readResult // non-nil if probeRequestBody called } -func newTransferWriter(r interface{}) (t *transferWriter, err error) { - t = &transferWriter{} +func http_newTransferWriter(r any) (t *http_transferWriter, err error) { + t = &http_transferWriter{} + // Extract relevant fields atLeastHTTP11 := false switch rr := r.(type) { - case *Request: + case *http_Request: if rr.ContentLength != 0 && rr.Body == nil { return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) } - t.Method = rr.Method - t.Body = rr.Body - t.BodyCloser = rr.Body - t.ContentLength = rr.ContentLength + t.Method = http_valueOrDefault(rr.Method, "GET") t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header t.Trailer = rr.Trailer - atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 { - if t.ContentLength == 0 { - // Test to see if it's actually zero or just unset. - var buf [1]byte - n, rerr := io.ReadFull(t.Body, buf[:]) - if rerr != nil && rerr != io.EOF { - t.ContentLength = -1 - t.Body = errorReader{rerr} - } else if n == 1 { - - t.ContentLength = -1 - t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body) - } else { - - t.Body = nil - t.BodyCloser = nil - } - } - if t.ContentLength < 0 { - t.TransferEncoding = []string{"chunked"} - } + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.outgoingLength() + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { + t.TransferEncoding = []string{"chunked"} } - case *Response: + // If there's a body, conservatively flush the headers + // to any bufio.Writer we're writing to, just in case + // the server needs the headers early, before we copy + // the body and possibly block. We make an exception + // for the common standard library in-memory types, + // though, to avoid unnecessary TCP packets on the + // wire. (Issue 22088.) + if t.ContentLength != 0 && !http_isKnownInMemoryReader(t.Body) { + t.FlushHeaders = true + } + + atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 + case *http_Response: t.IsResponse = true if rr.Request != nil { t.Method = rr.Request.Method @@ -5692,40 +13122,148 @@ t.ContentLength = rr.ContentLength t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - t.ResponseToHEAD = noBodyExpected(t.Method) + t.ResponseToHEAD = http_noResponseBodyExpected(t.Method) } + // Sanitize Body,ContentLength,TransferEncoding if t.ResponseToHEAD { t.Body = nil - if chunked(t.TransferEncoding) { + if http_chunked(t.TransferEncoding) { t.ContentLength = -1 } } else { if !atLeastHTTP11 || t.Body == nil { t.TransferEncoding = nil } - if chunked(t.TransferEncoding) { + if http_chunked(t.TransferEncoding) { t.ContentLength = -1 - } else if t.Body == nil { + } else if t.Body == nil { // no chunking, no body t.ContentLength = 0 } } - if !chunked(t.TransferEncoding) { + // Sanitize Trailer + if !http_chunked(t.TransferEncoding) { t.Trailer = nil } + // Validate Trailer names and values. The names are later written + // unmodified on the "Trailer:" line of the header, so invalid bytes + // (in particular CR and LF) would permit header injection. (Issue 78775.) + if err := http_validateHeaders(t.Trailer); err != "" { + return nil, fmt.Errorf("net/http: invalid trailer %s", err) + } + return t, nil } -func noBodyExpected(requestMethod string) bool { +// shouldSendChunkedRequestBody reports whether we should try to send a +// chunked request body to the server. In particular, the case we really +// want to prevent is sending a GET or other typically-bodyless request to a +// server with a chunked body when the body has zero bytes, since GETs with +// bodies (while acceptable according to specs), even zero-byte chunked +// bodies, are approximately never seen in the wild and confuse most +// servers. See Issue 18257, as one example. +// +// The only reason we'd send such a request is if the user set the Body to a +// non-nil value (say, io.NopCloser(bytes.NewReader(nil))) and didn't +// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume +// there's bytes to send. +// +// This code tries to read a byte from the Request.Body in such cases to see +// whether the body actually has content (super rare) or is actually just +// a non-nil content-less ReadCloser (the more common case). In that more +// common case, we act as if their Body were nil instead, and don't send +// a body. +func (t *http_transferWriter) shouldSendChunkedRequestBody() bool { + // Note that t.ContentLength is the corrected content length + // from rr.outgoingLength, so 0 actually means zero, not unknown. + if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if t.Method == "CONNECT" { + return false + } + if http_requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// - the request body blocks +// - the content length is not set (or set to -1) +// - the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// - there is no transfer-encoding=chunked already set. +// +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *http_transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan http_readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres http_readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + close(t.ByteReadCh) + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&http_byteReader{b: rres.b}, http_errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&http_byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = http_errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(http_finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func http_noResponseBodyExpected(requestMethod string) bool { return requestMethod == "HEAD" } -func (t *transferWriter) shouldSendContentLength() bool { - if chunked(t.TransferEncoding) { +func (t *http_transferWriter) shouldSendContentLength() bool { + if http_chunked(t.TransferEncoding) { return false } if t.ContentLength > 0 { @@ -5734,11 +13272,11 @@ if t.ContentLength < 0 { return false } - - if t.Method == "POST" || t.Method == "PUT" { + // Many servers expect a Content-Length for these methods + if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" { return true } - if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { + if t.ContentLength == 0 && http_isIdentity(t.TransferEncoding) { if t.Method == "GET" || t.Method == "HEAD" { return false } @@ -5748,13 +13286,19 @@ return false } -func (t *transferWriter) WriteHeader(w io.Writer) error { - if t.Close { +func (t *http_transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { + if t.Close && !http_hasToken(t.Header.get("Connection"), "close") { if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { return err } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Connection", []string{"close"}) + } } + // Write Content-Length and/or Transfer-Encoding whose values are a + // function of the sanitized field triple (Body, ContentLength, + // TransferEncoding) if t.shouldSendContentLength() { if _, err := io.WriteString(w, "Content-Length: "); err != nil { return err @@ -5762,63 +13306,95 @@ if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { return err } - } else if chunked(t.TransferEncoding) { + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) + } + } else if http_chunked(t.TransferEncoding) { if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { return err } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) + } } + // Write Trailer header if t.Trailer != nil { keys := make([]string, 0, len(t.Trailer)) for k := range t.Trailer { - k = CanonicalHeaderKey(k) + k = http_CanonicalHeaderKey(k) switch k { case "Transfer-Encoding", "Trailer", "Content-Length": - return &badStringError{"invalid Trailer key", k} + return http_badStringError("invalid Trailer key", k) } keys = append(keys, k) } if len(keys) > 0 { - sort.Strings(keys) - + slices.Sort(keys) + // TODO: could do better allocation-wise here, but trailers are rare, + // so being lazy for now. if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { return err } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Trailer", keys) + } } } return nil } -func (t *transferWriter) WriteBody(w io.Writer) error { - var err error +// always closes t.BodyCloser +func (t *http_transferWriter) writeBody(w io.Writer) (err error) { var ncopy int64 + closed := false + defer func() { + if closed || t.BodyCloser == nil { + return + } + if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() - if t.Body != nil { - if chunked(t.TransferEncoding) { + // Write body. We "unwrap" the body first if it was wrapped in a + // nopCloser or readTrackingBody. This is to ensure that we can take advantage of + // OS-level optimizations in the event that the body is an + // *os.File. + if !t.ResponseToHEAD && t.Body != nil { + var body = t.unwrapBody() + if http_chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { - w = &internal.FlushAfterChunkWriter{bw} + w = &internal.FlushAfterChunkWriter{Writer: bw} } cw := internal.NewChunkedWriter(w) - _, err = io.Copy(cw, t.Body) + _, err = t.doBodyCopy(cw, body) if err == nil { err = cw.Close() } } else if t.ContentLength == -1 { - ncopy, err = io.Copy(w, t.Body) + dst := w + if t.Method == "CONNECT" { + dst = http_bufioFlushWriter{dst} + } + ncopy, err = t.doBodyCopy(dst, body) } else { - ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) + ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) if err != nil { return err } var nextra int64 - nextra, err = io.Copy(io.Discard, t.Body) + nextra, err = t.doBodyCopy(io.Discard, body) ncopy += nextra } if err != nil { return err } - if err = t.BodyCloser.Close(); err != nil { + } + if t.BodyCloser != nil { + closed = true + if err := t.BodyCloser.Close(); err != nil { return err } } @@ -5828,37 +13404,72 @@ t.ContentLength, ncopy) } - if chunked(t.TransferEncoding) { - + if !t.ResponseToHEAD && http_chunked(t.TransferEncoding) { + // Write Trailer header if t.Trailer != nil { if err := t.Trailer.Write(w); err != nil { return err } } - + // Last chunk, empty trailer _, err = io.WriteString(w, "\r\n") } return err } -type transferReader struct { +// doBodyCopy wraps a copy operation, with any resulting error also +// being saved in bodyReadError. +// +// This function is only intended for use in writeBody. +func (t *http_transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { + buf := http_getCopyBuf() + defer http_putCopyBuf(buf) + + n, err = io.CopyBuffer(dst, src, buf) + if err != nil && err != io.EOF { + t.bodyReadError = err + } + return +} + +// unwrapBody unwraps the body's inner reader if it's a +// nopCloser. This is to ensure that body writes sourced from local +// files (*os.File types) are properly optimized. +// +// This function is only intended for use in writeBody. +func (t *http_transferWriter) unwrapBody() io.Reader { + if r, ok := http_unwrapNopCloser(t.Body); ok { + return r + } + if r, ok := t.Body.(*http_readTrackingBody); ok { + r.didRead = true + return r.ReadCloser + } + return t.Body +} + +type http_transferReader struct { // Input - Header Header + Header http_Header StatusCode int RequestMethod string ProtoMajor int ProtoMinor int // Output - Body io.ReadCloser - ContentLength int64 - TransferEncoding []string - Close bool - Trailer Header + Body io.ReadCloser + ContentLength int64 + Chunked bool + Close bool + Trailer http_Header +} + +func (t *http_transferReader) protoAtLeast(m, n int) bool { + return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. -func bodyAllowedForStatus(status int) bool { +// permits a body. See RFC 7230, section 3.3. +func http_bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: return false @@ -5871,64 +13482,68 @@ } var ( - suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} - suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} + http_suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} + http_suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} + http_excludedHeadersNoBody = map[string]bool{"Content-Length": true, "Transfer-Encoding": true} ) -func suppressedHeaders(status int) []string { +func http_suppressedHeaders(status int) []string { switch { case status == 304: - - return suppressedHeaders304 - case !bodyAllowedForStatus(status): - return suppressedHeadersNoBody + // RFC 7232 section 4.1 + return http_suppressedHeaders304 + case !http_bodyAllowedForStatus(status): + return http_suppressedHeadersNoBody } return nil } // msg is *Request or *Response. -func readTransfer(msg interface{}, r *bufio.Reader) (err error) { - t := &transferReader{RequestMethod: "GET"} +func http_readTransfer(msg any, r *bufio.Reader, maxTrailerHeaders int64) (err error) { + t := &http_transferReader{RequestMethod: "GET"} + // Unify input isResponse := false switch rr := msg.(type) { - case *Response: + case *http_Response: t.Header = rr.Header t.StatusCode = rr.StatusCode t.ProtoMajor = rr.ProtoMajor t.ProtoMinor = rr.ProtoMinor - t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) + t.Close = http_shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) isResponse = true if rr.Request != nil { t.RequestMethod = rr.Request.Method } - case *Request: + case *http_Request: t.Header = rr.Header t.RequestMethod = rr.Method t.ProtoMajor = rr.ProtoMajor t.ProtoMinor = rr.ProtoMinor - + // Transfer semantics for Requests are exactly like those for + // Responses with status code 200, responding to a GET method t.StatusCode = 200 t.Close = rr.Close default: panic("unexpected type") } + // Default to HTTP/1.1 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { t.ProtoMajor, t.ProtoMinor = 1, 1 } - t.TransferEncoding, err = fixTransferEncoding(isResponse, t.RequestMethod, t.Header) - if err != nil { + // Transfer-Encoding: chunked, and overriding Content-Length. + if err := t.parseTransferEncoding(); err != nil { return err } - realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding) + realLength, err := http_fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) if err != nil { return err } if isResponse && t.RequestMethod == "HEAD" { - if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { + if n, err := http_parseContentLength(t.Header["Content-Length"]); err != nil { return err } else { t.ContentLength = n @@ -5937,54 +13552,63 @@ t.ContentLength = realLength } - t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding) + // Trailer + t.Trailer, err = http_fixTrailer(t.Header, t.Chunked) if err != nil { return err } + // If there is no Content-Length or chunked Transfer-Encoding on a *Response + // and the status is not 1xx, 204 or 304, then the body is unbounded. + // See RFC 7230, section 3.3. switch msg.(type) { - case *Response: - if realLength == -1 && - !chunked(t.TransferEncoding) && - bodyAllowedForStatus(t.StatusCode) { - + case *http_Response: + if realLength == -1 && !t.Chunked && http_bodyAllowedForStatus(t.StatusCode) { + // Unbounded body. t.Close = true } } + // Prepare body reader. ContentLength < 0 means chunked encoding + // or close connection when finished, since multipart is not supported yet switch { - case chunked(t.TransferEncoding): - if noBodyExpected(t.RequestMethod) { - t.Body = eofReader + case t.Chunked: + if isResponse && (http_noResponseBodyExpected(t.RequestMethod) || !http_bodyAllowedForStatus(t.StatusCode)) { + t.Body = http_NoBody } else { - t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} + t.Body = &http_body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close, maxTrailerHeaders: maxTrailerHeaders} } case realLength == 0: - t.Body = eofReader + t.Body = http_NoBody case realLength > 0: - t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} + t.Body = &http_body{src: io.LimitReader(r, realLength), closing: t.Close} default: - + // realLength < 0, i.e. "Content-Length" not mentioned in header if t.Close { - - t.Body = &body{src: r, closing: t.Close} + // Close semantics (i.e. HTTP/1.0) + t.Body = &http_body{src: r, closing: t.Close} } else { - - t.Body = eofReader + // Persistent connection (i.e. HTTP/1.1) + t.Body = http_NoBody } } + // Unify output switch rr := msg.(type) { - case *Request: + case *http_Request: rr.Body = t.Body rr.ContentLength = t.ContentLength - rr.TransferEncoding = t.TransferEncoding + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } rr.Close = t.Close rr.Trailer = t.Trailer - case *Response: + case *http_Response: rr.Body = t.Body rr.ContentLength = t.ContentLength - rr.TransferEncoding = t.TransferEncoding + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } rr.Close = t.Close rr.Trailer = t.Trailer } @@ -5992,59 +13616,93 @@ return nil } -// Checks whether chunked is part of the encodings stack -func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } +// Checks whether chunked is part of the encodings stack. +func http_chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Checks whether the encoding is explicitly "identity". -func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } +func http_isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } -// Sanitize transfer encoding -func fixTransferEncoding(isResponse bool, requestMethod string, header Header) ([]string, error) { - raw, present := header["Transfer-Encoding"] - if !present { - return nil, nil - } - delete(header, "Transfer-Encoding") - - encodings := strings.Split(raw[0], ",") - te := make([]string, 0, len(encodings)) - - for _, encoding := range encodings { - encoding = strings.ToLower(strings.TrimSpace(encoding)) - - if encoding == "identity" { - break - } - if encoding != "chunked" { - return nil, &badStringError{"unsupported transfer encoding", encoding} - } - te = te[0 : len(te)+1] - te[len(te)-1] = encoding - } - if len(te) > 1 { - return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")} - } - if len(te) > 0 { - - delete(header, "Content-Length") - return te, nil - } - - return nil, nil +// unsupportedTEError reports unsupported transfer-encodings. +type http_unsupportedTEError struct { + err string } -// Determine the expected body length, using RFC 2616 Section 4.4. This +func (uste *http_unsupportedTEError) Error() string { + return uste.err +} + +// isUnsupportedTEError checks if the error is of type +// unsupportedTEError. It is usually invoked with a non-nil err. +func http_isUnsupportedTEError(err error) bool { + _, ok := err.(*http_unsupportedTEError) + return ok +} + +// parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. +func (t *http_transferReader) parseTransferEncoding() error { + raw, present := t.Header["Transfer-Encoding"] + if !present { + return nil + } + delete(t.Header, "Transfer-Encoding") + + // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. + if !t.protoAtLeast(1, 1) { + return nil + } + + // Like nginx, we only support a single Transfer-Encoding header field, and + // only if set to "chunked". This is one of the most security sensitive + // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it + // strict and simple. + if len(raw) != 1 { + return &http_unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)} + } + if !ascii.EqualFold(raw[0], "chunked") { + return &http_unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])} + } + + t.Chunked = true + return nil +} + +// Determine the expected body length, using RFC 7230 Section 3.3. This // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. -func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { - contentLens := header["Content-Length"] +func http_fixLength(isResponse bool, status int, requestMethod string, header http_Header, chunked bool) (n int64, err error) { isRequest := !isResponse + contentLens := header["Content-Length"] - if noBodyExpected(requestMethod) { - - if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { - return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) + // Hardening against HTTP request smuggling + if len(contentLens) > 1 { + // Per RFC 7230 Section 3.3.2, prevent multiple + // Content-Length headers if they differ in value. + // If there are dups of the value, remove the dups. + // See Issue 16490. + first := textproto.TrimString(contentLens[0]) + for _, ct := range contentLens[1:] { + if first != textproto.TrimString(ct) { + return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) + } } + + // deduplicate Content-Length + header.Del("Content-Length") + header.Add("Content-Length", first) + + contentLens = header["Content-Length"] + } + + // Reject requests with invalid Content-Length headers. + if len(contentLens) > 0 { + n, err = http_parseContentLength(contentLens) + if err != nil { + return -1, err + } + } + + // Logic based on response type or status + if isResponse && http_noResponseBodyExpected(requestMethod) { return 0, nil } if status/100 == 1 { @@ -6055,122 +13713,147 @@ return 0, nil } - if len(contentLens) > 1 { - - return 0, errors.New("http: message cannot contain multiple Content-Length headers") - } - - if chunked(te) { + // According to RFC 9112, "If a message is received with both a + // Transfer-Encoding and a Content-Length header field, the Transfer-Encoding + // overrides the Content-Length. Such a message might indicate an attempt to + // perform request smuggling (Section 11.2) or response splitting (Section 11.1) + // and ought to be handled as an error. An intermediary that chooses to forward + // the message MUST first remove the received Content-Length field and process + // the Transfer-Encoding (as described below) prior to forwarding the message downstream." + // + // Chunked-encoding requests with either valid Content-Length + // headers or no Content-Length headers are accepted after removing + // the Content-Length field from header. + // + // Logic based on Transfer-Encoding + if chunked { + header.Del("Content-Length") return -1, nil } // Logic based on Content-Length - var cl string - if len(contentLens) == 1 { - cl = strings.TrimSpace(contentLens[0]) - } - if cl != "" { - n, err := parseContentLength(cl) - if err != nil { - return -1, err - } + if len(contentLens) > 0 { return n, nil - } else { - header.Del("Content-Length") } - if !isResponse { + header.Del("Content-Length") + if isRequest { + // RFC 7230 neither explicitly permits nor forbids an + // entity-body on a GET request so we permit one if + // declared, but we default to 0 here (not -1 below) + // if there's no mention of a body. + // Likewise, all other request methods are assumed to have + // no body if neither Transfer-Encoding chunked nor a + // Content-Length are set. return 0, nil } + // Body-EOF logic based on other methods (like closing, or chunked coding) return -1, nil } // Determine whether to hang up after sending a request and body, or // receiving a response and body -// 'header' is the request headers -func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { +// 'header' is the request headers. +func http_shouldClose(major, minor int, header http_Header, removeCloseHeader bool) bool { if major < 1 { return true - } else if major == 1 && minor == 0 { - vv := header["Connection"] - if headerValuesContainsToken(vv, "close") || !headerValuesContainsToken(vv, "keep-alive") { - return true - } - return false - } else { - if headerValuesContainsToken(header["Connection"], "close") { - if removeCloseHeader { - header.Del("Connection") - } - return true - } } - return false + + conv := header["Connection"] + hasClose := httpguts.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose } -// Parse the trailer header -func fixTrailer(header Header, te []string) (Header, error) { - raw := header.get("Trailer") - if raw == "" { +// Parse the trailer header. +func http_fixTrailer(header http_Header, chunked bool) (http_Header, error) { + vv, ok := header["Trailer"] + if !ok { return nil, nil } - + if !chunked { + // Trailer and no chunking: + // this is an invalid use case for trailer header. + // Nevertheless, no error will be returned and we + // let users decide if this is a valid HTTP message. + // The Trailer header will be kept in Response.Header + // but not populate Response.Trailer. + // See issue #27197. + return nil, nil + } header.Del("Trailer") - trailer := make(Header) - keys := strings.Split(raw, ",") - for _, key := range keys { - key = CanonicalHeaderKey(strings.TrimSpace(key)) - switch key { - case "Transfer-Encoding", "Trailer", "Content-Length": - return nil, &badStringError{"bad trailer key", key} - } - trailer[key] = nil + + trailer := make(http_Header) + var err error + for _, v := range vv { + http_foreachHeaderElement(v, func(key string) { + key = http_CanonicalHeaderKey(key) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + if err == nil { + err = http_badStringError("bad trailer key", key) + return + } + } + trailer[key] = nil + }) + } + if err != nil { + return nil, err } if len(trailer) == 0 { return nil, nil } - if !chunked(te) { - - return nil, ErrUnexpectedTrailer - } return trailer, nil } // body turns a Reader into a ReadCloser. // Close ensures that the body has been fully read // and then reads the trailer if necessary. -type body struct { - src io.Reader - hdr interface{} // non-nil (Response or Request) value means read trailer - r *bufio.Reader // underlying wire-format reader for the trailer - closing bool // is the connection to be closed after reading body? - doEarlyClose bool // whether Close should stop early +type http_body struct { + src io.Reader + hdr any // non-nil (Response or Request) value means read trailer + r *bufio.Reader // underlying wire-format reader for the trailer + closing bool // is the connection to be closed after reading body? + doEarlyClose bool // whether Close should stop early + maxTrailerHeaders int64 // how many trailer header values are allowed - mu sync.Mutex // guards closed, and calls to Read and Close + mu sync.Mutex // guards following, and calls to Read and Close sawEOF bool closed bool - earlyClose bool // Close called and we didn't read to the end of src + earlyClose bool // Close called and we didn't read to the end of src + onHitEOF func() // if non-nil, func to call when EOF is Read } -// ErrBodyReadAfterClose is returned when reading a Request or Response +// ErrBodyReadAfterClose is returned when reading a [Request] or [Response] // Body after the body has been closed. This typically happens when the body is -// read after an HTTP Handler calls WriteHeader or Write on its -// ResponseWriter. -var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") +// read after an HTTP [Handler] calls WriteHeader or Write on its +// [ResponseWriter]. +var http_ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") -func (b *body) Read(p []byte) (n int, err error) { +func (b *http_body) Read(p []byte) (n int, err error) { + if b == nil { + return 0, io.EOF + } b.mu.Lock() defer b.mu.Unlock() if b.closed { - return 0, ErrBodyReadAfterClose + return 0, http_ErrBodyReadAfterClose } return b.readLocked(p) } // Must hold b.mu. -func (b *body) readLocked(p []byte) (n int, err error) { +func (b *http_body) readLocked(p []byte) (n int, err error) { if b.sawEOF { return 0, io.EOF } @@ -6178,23 +13861,32 @@ if err == io.EOF { b.sawEOF = true - + // Chunked case. Read the trailer. if b.hdr != nil { if e := b.readTrailer(); e != nil { err = e - + // Something went wrong in the trailer, we must not allow any + // further reads of any kind to succeed from body, nor any + // subsequent requests on the server connection. See + // golang.org/issue/12027 b.sawEOF = false b.closed = true } b.hdr = nil } else { - + // If the server declared the Content-Length, our body is a LimitedReader + // and we need to check whether this EOF arrived early. if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { err = io.ErrUnexpectedEOF } } } + // If we can return an EOF here along with the read data, do + // so. This is optional per the io.Reader contract, but doing + // so helps the HTTP transport code recycle its connection + // earlier (since it will see this EOF itself), even if the + // client doesn't do future reads or Close. if err == nil && n > 0 { if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { err = io.EOF @@ -6202,19 +13894,24 @@ } } + if b.sawEOF && b.onHitEOF != nil { + b.onHitEOF() + } + return n, err } var ( - singleCRLF = []byte("\r\n") - doubleCRLF = []byte("\r\n\r\n") + http_singleCRLF = []byte("\r\n") + http_doubleCRLF = []byte("\r\n\r\n") ) -func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { +func http_seeUpcomingDoubleCRLF(r *bufio.Reader) bool { for peekSize := 4; ; peekSize++ { - + // This loop stops when Peek returns an error, + // which it does when r's buffer has been filled. buf, err := r.Peek(peekSize) - if bytes.HasSuffix(buf, doubleCRLF) { + if bytes.HasSuffix(buf, http_doubleCRLF) { return true } if err != nil { @@ -6224,63 +13921,72 @@ return false } -var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") +var http_errTrailerEOF = errors.New("http: unexpected EOF reading trailer") -func (b *body) readTrailer() error { - +func (b *http_body) readTrailer() error { + // The common case, since nobody uses trailers. buf, err := b.r.Peek(2) - if bytes.Equal(buf, singleCRLF) { + if bytes.Equal(buf, http_singleCRLF) { b.r.Discard(2) return nil } if len(buf) < 2 { - return errTrailerEOF + return http_errTrailerEOF } if err != nil { return err } - if !seeUpcomingDoubleCRLF(b.r) { + // Make sure there's a header terminator coming up, to prevent + // a DoS with an unbounded size Trailer. It's not easy to + // slip in a LimitReader here, as textproto.NewReader requires + // a concrete *bufio.Reader. Also, we can't get all the way + // back up to our conn's LimitedReader that *might* be backing + // this bufio.Reader. Instead, a hack: we iteratively Peek up + // to the bufio.Reader's max size, looking for a double CRLF. + // This limits the trailer to the underlying buffer size, typically 4kB. + if !http_seeUpcomingDoubleCRLF(b.r) { return errors.New("http: suspiciously long trailer after chunked body") } - hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() + hdr, err := http_readMIMEHeader(textproto.NewReader(b.r), math.MaxInt64, b.maxTrailerHeaders) if err != nil { if err == io.EOF { - return errTrailerEOF + return http_errTrailerEOF } return err } switch rr := b.hdr.(type) { - case *Request: - mergeSetHeader(&rr.Trailer, Header(hdr)) - case *Response: - mergeSetHeader(&rr.Trailer, Header(hdr)) + case *http_Request: + http_mergeSetHeader(&rr.Trailer, http_Header(hdr)) + case *http_Response: + http_mergeSetHeader(&rr.Trailer, http_Header(hdr)) } return nil } -func mergeSetHeader(dst *Header, src Header) { +func http_mergeSetHeader(dst *http_Header, src http_Header) { if *dst == nil { *dst = src return } - for k, vv := range src { - (*dst)[k] = vv - } + maps.Copy(*dst, src) } // unreadDataSizeLocked returns the number of bytes of unread input. // It returns -1 if unknown. // b.mu must be held. -func (b *body) unreadDataSizeLocked() int64 { +func (b *http_body) unreadDataSizeLocked() int64 { if lr, ok := b.src.(*io.LimitedReader); ok { return lr.N } return -1 } -func (b *body) Close() error { +func (b *http_body) Close() error { + if b == nil { + return nil + } b.mu.Lock() defer b.mu.Unlock() if b.closed { @@ -6289,132 +13995,330 @@ var err error switch { case b.sawEOF: - + // Already saw EOF, so no need going to look for it. case b.hdr == nil && b.closing: - + // no trailer and closing the connection next. + // no point in reading to EOF. case b.doEarlyClose: - - if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { - + // Read up to maxPostHandlerReadBytes bytes of the body, looking + // for EOF (and trailers), so we can re-use this connection. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > http_maxPostHandlerReadBytes { + // There was a declared Content-Length, and we have more bytes remaining + // than our maxPostHandlerReadBytes tolerance. So, give up. b.earlyClose = true } else { var n int64 - - n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes) - if err == io.EOF { - err = nil - } - if n == maxPostHandlerReadBytes { - b.earlyClose = true + // Consume the body, or, which will also lead to us reading + // the trailer headers after the body, if present. + n, err = io.CopyN(io.Discard, http_bodyLocked{b}, http_maxPostHandlerReadBytes+1) + b.earlyClose = true + if err == io.EOF && n <= http_maxPostHandlerReadBytes { + b.earlyClose = false + b.sawEOF = true } } default: - - _, err = io.Copy(io.Discard, bodyLocked{b}) + // Fully consume the body, which will also lead to us reading + // the trailer headers after the body, if present. + _, err = io.Copy(io.Discard, http_bodyLocked{b}) } b.closed = true return err } -func (b *body) didEarlyClose() bool { +func (b *http_body) didEarlyClose() bool { b.mu.Lock() defer b.mu.Unlock() return b.earlyClose } -// bodyLocked is a io.Reader reading from a *body when its mutex is -// already held. -type bodyLocked struct { - b *body +// bodyRemains reports whether future Read calls might +// yield data. +func (b *http_body) bodyRemains() bool { + if b == nil { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + return !b.sawEOF } -func (bl bodyLocked) Read(p []byte) (n int, err error) { +func (b *http_body) registerOnHitEOF(fn func()) { + if b == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + b.onHitEOF = fn +} + +// bodyLocked is an io.Reader reading from a *body when its mutex is +// already held. +type http_bodyLocked struct { + b *http_body +} + +func (bl http_bodyLocked) Read(p []byte) (n int, err error) { if bl.b.closed { - return 0, ErrBodyReadAfterClose + return 0, http_ErrBodyReadAfterClose } return bl.b.readLocked(p) } -// parseContentLength trims whitespace from s and returns -1 if no value -// is set, or the value if it's >= 0. -func parseContentLength(cl string) (int64, error) { - cl = strings.TrimSpace(cl) - if cl == "" { +var http_httplaxcontentlength = godebug.New("httplaxcontentlength") + +// parseContentLength checks that the header is valid and then trims +// whitespace. It returns -1 if no value is set otherwise the value +// if it's >= 0. +func http_parseContentLength(clHeaders []string) (int64, error) { + if len(clHeaders) == 0 { return -1, nil } - n, err := strconv.ParseInt(cl, 10, 64) - if err != nil || n < 0 { - return 0, &badStringError{"bad Content-Length", cl} - } - return n, nil + cl := textproto.TrimString(clHeaders[0]) + // The Content-Length must be a valid numeric value. + // See: https://datatracker.ietf.org/doc/html/rfc2616/#section-14.13 + if cl == "" { + if http_httplaxcontentlength.Value() == "1" { + http_httplaxcontentlength.IncNonDefault() + return -1, nil + } + return 0, http_badStringError("invalid empty Content-Length", cl) + } + n, err := strconv.ParseUint(cl, 10, 63) + if err != nil { + return 0, http_badStringError("bad Content-Length", cl) + } + return int64(n), nil } -// DefaultTransport is the default implementation of Transport and is -// used by DefaultClient. It establishes network connections as needed +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type http_finishAsyncByteRead struct { + tw *http_transferWriter +} + +func (fr http_finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + if err == nil { + err = io.EOF + } + return +} + +var http_nopCloserType = reflect.TypeOf(io.NopCloser(nil)) + +var http_nopCloserWriterToType = reflect.TypeOf(io.NopCloser(struct { + io.Reader + io.WriterTo +}{})) + +// unwrapNopCloser return the underlying reader and true if r is a NopCloser +// else it return false. +func http_unwrapNopCloser(r io.Reader) (underlyingReader io.Reader, isNopCloser bool) { + switch reflect.TypeOf(r) { + case http_nopCloserType, http_nopCloserWriterToType: + return reflect.ValueOf(r).Field(0).Interface().(io.Reader), true + default: + return nil, false + } +} + +// isKnownInMemoryReader reports whether r is a type known to not +// block on Read. Its caller uses this as an optional optimization to +// send fewer TCP packets. +func http_isKnownInMemoryReader(r io.Reader) bool { + switch r.(type) { + case *bytes.Reader, *bytes.Buffer, *strings.Reader: + return true + } + if r, ok := http_unwrapNopCloser(r); ok { + return http_isKnownInMemoryReader(r) + } + if r, ok := r.(*http_readTrackingBody); ok { + return http_isKnownInMemoryReader(r.ReadCloser) + } + return false +} + +// bufioFlushWriter is an io.Writer wrapper that flushes all writes +// on its wrapped writer if it's a *bufio.Writer. +type http_bufioFlushWriter struct{ w io.Writer } + +func (fw http_bufioFlushWriter) Write(p []byte) (n int, err error) { + n, err = fw.w.Write(p) + if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { + ferr := bw.Flush() + if ferr != nil && err == nil { + err = ferr + } + } + return +} + +// DefaultTransport is the default implementation of [Transport] and is +// used by [DefaultClient]. It establishes network connections as needed // and caches them for reuse by subsequent calls. It uses HTTP proxies -// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and -// $no_proxy) environment variables. -var DefaultTransport RoundTripper = &Transport{ - Proxy: ProxyFromEnvironment, - Dial: (&net.Dialer{ +// as directed by the environment variables HTTP_PROXY, HTTPS_PROXY +// and NO_PROXY (or the lowercase versions thereof, which take +// precedence over the uppercase versions). +var http_DefaultTransport http_RoundTripper = &http_Transport{ + Proxy: http_ProxyFromEnvironment, + DialContext: http_defaultTransportDialContext(&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, + }), + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, } -// DefaultMaxIdleConnsPerHost is the default value of Transport's +// DefaultMaxIdleConnsPerHost is the default value of [Transport]'s // MaxIdleConnsPerHost. -const DefaultMaxIdleConnsPerHost = 2 +const http_DefaultMaxIdleConnsPerHost = 2 -// Transport is an implementation of RoundTripper that supports HTTP, +// Transport is an implementation of [RoundTripper] that supports HTTP, // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). -// Transport can also cache connections for future re-use. -type Transport struct { - idleMu sync.Mutex - wantIdle bool // user has requested to close all idle conns - idleConn map[connectMethodKey][]*persistConn - idleConnCh map[connectMethodKey]chan *persistConn +// +// By default, Transport caches connections for future re-use. +// This may leave many open connections when accessing many hosts. +// This behavior can be managed using [Transport.CloseIdleConnections] method +// and the [Transport.MaxIdleConnsPerHost] and [Transport.DisableKeepAlives] fields. +// +// Transports should be reused instead of created as needed. +// Transports are safe for concurrent use by multiple goroutines. +// +// A Transport is a low-level primitive for making HTTP and HTTPS requests. +// For high-level functionality, such as cookies and redirects, see [Client]. +// +// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 +// for HTTPS URLs, depending on whether the server supports HTTP/2, +// and how the Transport is configured. The [DefaultTransport] supports HTTP/2. +// To explicitly enable HTTP/2 on a transport, set [Transport.Protocols]. +// +// Responses with status codes in the 1xx range are either handled +// automatically (100 expect-continue) or ignored. The one +// exception is HTTP status code 101 (Switching Protocols), which is +// considered a terminal status and returned by [Transport.RoundTrip]. To see the +// ignored 1xx responses, use the httptrace trace package's +// ClientTrace.Got1xxResponse. +// +// Transport only retries a request upon encountering a network error +// if the connection has already been used successfully and if the +// request is idempotent and either has no body or has its [Request.GetBody] +// defined. HTTP requests are considered idempotent if they have HTTP methods +// GET, HEAD, OPTIONS, or TRACE; or if their [Header] map contains an +// "Idempotency-Key" or "X-Idempotency-Key" entry. If the idempotency key +// value is a zero-length slice, the request is treated as idempotent but the +// header is not sent on the wire. +type http_Transport struct { + idleMu sync.Mutex + closeIdle bool // user has requested to close all idle conns + idleConn map[http_connectMethodKey][]*http_persistConn // most recently used at end + idleConnWait map[http_connectMethodKey]http_wantConnQueue // waiting getConns + idleLRU http_connLRU reqMu sync.Mutex - reqCanceler map[*Request]func() + reqCanceler map[*http_Request]context.CancelCauseFunc - altMu sync.RWMutex - altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper + altMu sync.Mutex // guards changing altProto only + altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme + + connsPerHostMu sync.Mutex + connsPerHost map[http_connectMethodKey]int + connsPerHostWait map[http_connectMethodKey]http_wantConnQueue // waiting getConns + dialsInProgress http_wantConnQueue // Proxy specifies a function to return a proxy for a given // Request. If the function returns a non-nil error, the // request is aborted with the provided error. + // + // The proxy type is determined by the URL scheme. "http", + // "https", "socks5", and "socks5h" are supported. If the scheme is empty, + // "http" is assumed. + // "socks5" is treated the same as "socks5h". + // + // If the proxy URL contains a userinfo subcomponent, + // the proxy request will pass the username and password + // in a Proxy-Authorization header. + // // If Proxy is nil or returns a nil *URL, no proxy is used. - Proxy func(*Request) (*url.URL, error) + Proxy func(*http_Request) (*url.URL, error) - // Dial specifies the dial function for creating unencrypted - // TCP connections. - // If Dial is nil, net.Dial is used. + // OnProxyConnectResponse is called when the Transport gets an HTTP response from + // a proxy for a CONNECT request. It's called before the check for a 200 OK response. + // If it returns an error, the request fails with that error. + OnProxyConnectResponse func(ctx context.Context, proxyURL *url.URL, connectReq *http_Request, connectRes *http_Response) error + + // DialContext specifies the dial function for creating unencrypted TCP connections. + // If DialContext is nil (and the deprecated Dial below is also nil), + // then the transport dials using package net. + // + // DialContext runs concurrently with calls to RoundTrip. + // A RoundTrip call that initiates a dial may end up using + // a connection dialed previously when the earlier connection + // becomes idle before the later DialContext completes. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Dial specifies the dial function for creating unencrypted TCP connections. + // + // Dial runs concurrently with calls to RoundTrip. + // A RoundTrip call that initiates a dial may end up using + // a connection dialed previously when the earlier connection + // becomes idle before the later Dial completes. + // + // Deprecated: Use DialContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialContext takes priority. Dial func(network, addr string) (net.Conn, error) + // DialTLSContext specifies an optional dial function for creating + // TLS connections for non-proxied HTTPS requests. + // + // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), + // DialContext and TLSClientConfig are used. + // + // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS + // requests and the TLSClientConfig and TLSHandshakeTimeout + // are ignored. The returned net.Conn is assumed to already be + // past the TLS handshake. + // + // To support ALPN protocol negotiation, the returned net.Conn should be + // a *tls.Conn or implement the same ConnectionState method as *tls.Conn. + DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) + // DialTLS specifies an optional dial function for creating // TLS connections for non-proxied HTTPS requests. // - // If DialTLS is nil, Dial and TLSClientConfig are used. - // - // If DialTLS is set, the Dial hook is not used for HTTPS - // requests and the TLSClientConfig and TLSHandshakeTimeout - // are ignored. The returned net.Conn is assumed to already be - // past the TLS handshake. + // Deprecated: Use DialTLSContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialTLSContext takes priority. DialTLS func(network, addr string) (net.Conn, error) // TLSClientConfig specifies the TLS configuration to use with - // tls.Client. If nil, the default configuration is used. + // tls.Client. + // If nil, the default configuration is used. + // If non-nil, HTTP/2 support may not be enabled by default. TLSClientConfig *tls.Config - // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // TLSHandshakeTimeout specifies the maximum amount of time to // wait for a TLS handshake. Zero means no timeout. TLSHandshakeTimeout time.Duration - // DisableKeepAlives, if true, prevents re-use of TCP connections - // between different HTTP requests. + // DisableKeepAlives, if true, disables HTTP keep-alives and + // will only use the connection to the server for a single + // HTTP request. + // + // This is unrelated to the similarly named TCP keep-alives. DisableKeepAlives bool // DisableCompression, if true, prevents the Transport from @@ -6427,23 +14331,306 @@ // uncompressed. DisableCompression bool + // MaxIdleConns controls the maximum number of idle (keep-alive) + // connections across all hosts. Zero means no limit. + MaxIdleConns int + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle - // (keep-alive) to keep per-host. If zero, + // (keep-alive) connections to keep per-host. If zero, // DefaultMaxIdleConnsPerHost is used. MaxIdleConnsPerHost int + // MaxConnsPerHost optionally limits the total number of + // connections per host, including connections in the dialing, + // active, and idle states. On limit violation, dials will block. + // + // Zero means no limit. + MaxConnsPerHost int + + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself. + // Zero means no limit. + IdleConnTimeout time.Duration + // ResponseHeaderTimeout, if non-zero, specifies the amount of // time to wait for a server's response headers after fully // writing the request (including its body, if any). This // time does not include the time to read the response body. ResponseHeaderTimeout time.Duration + + // ExpectContinueTimeout, if non-zero, specifies the amount of + // time to wait for a server's first response headers after fully + // writing the request headers if the request has an + // "Expect: 100-continue" header. Zero means no timeout and + // causes the body to be sent immediately, without + // waiting for the server to approve. + // This time does not include the time to send the request header. + ExpectContinueTimeout time.Duration + + // TLSNextProto specifies how the Transport switches to an + // alternate protocol (such as HTTP/2) after a TLS ALPN + // protocol negotiation. If Transport dials a TLS connection + // with a non-empty protocol name and TLSNextProto contains a + // map entry for that key (such as "h2"), then the func is + // called with the request's authority (such as "example.com" + // or "example.com:1234") and the TLS connection. The function + // must return a RoundTripper that then handles the request. + // If TLSNextProto is not nil, HTTP/2 support is not enabled + // automatically. + // + // Historically, TLSNextProto was used to disable HTTP/2 support. + // The Transport.Protocols field now provides a simpler way to do this. + TLSNextProto map[string]func(authority string, c *tls.Conn) http_RoundTripper + + // ProxyConnectHeader optionally specifies headers to send to + // proxies during CONNECT requests. + // To set the header dynamically, see GetProxyConnectHeader. + ProxyConnectHeader http_Header + + // GetProxyConnectHeader optionally specifies a func to return + // headers to send to proxyURL during a CONNECT request to the + // ip:port target. + // If it returns an error, the Transport's RoundTrip fails with + // that error. It can return (nil, nil) to not add headers. + // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is + // ignored. + GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (http_Header, error) + + // MaxResponseHeaderBytes specifies a limit on how many + // response bytes are allowed in the server's response + // header. + // + // Zero means to use a default limit. + MaxResponseHeaderBytes int64 + + // WriteBufferSize specifies the size of the write buffer used + // when writing to the transport. + // If zero, a default (currently 4KB) is used. + WriteBufferSize int + + // ReadBufferSize specifies the size of the read buffer used + // when reading from the transport. + // If zero, a default (currently 4KB) is used. + ReadBufferSize int + + // nextProtoOnce guards initialization of TLSNextProto and + // h2Transport (via onceSetNextProtoDefaults) + nextProtoOnce sync.Once + closeIdleFunc http_closeIdleConnectionser // non-nil if http2 wired up + h2Transport *http_http2Transport + h2Config http_http2ExternalTransportConfig + h3Transport http_dialClientConner // non-nil if http3 wired up + tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired + + // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero + // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. + // By default, use of any those fields conservatively disables HTTP/2. + // To use a custom dialer or TLS config and still attempt HTTP/2 + // upgrades, set this to true. + ForceAttemptHTTP2 bool + + // HTTP2 configures HTTP/2 connections. + HTTP2 *http_HTTP2Config + + // Protocols is the set of protocols supported by the transport. + // + // If Protocols includes UnencryptedHTTP2 and does not include HTTP1, + // the transport will use unencrypted HTTP/2 for requests for http:// URLs. + // + // If Protocols is nil, the default is usually HTTP/1 only. + // If ForceAttemptHTTP2 is true, or if TLSNextProto contains an "h2" entry, + // the default is HTTP/1 and HTTP/2. + Protocols *http_Protocols +} + +func (t *http_Transport) writeBufferSize() int { + if t.WriteBufferSize > 0 { + return t.WriteBufferSize + } + return 4 << 10 +} + +func (t *http_Transport) readBufferSize() int { + if t.ReadBufferSize > 0 { + return t.ReadBufferSize + } + return 4 << 10 +} + +func (t *http_Transport) maxHeaderResponseSize() int64 { + if t.MaxResponseHeaderBytes > 0 { + return t.MaxResponseHeaderBytes + } + return 10 << 20 // conservative default; same as http2 +} + +// Clone returns a deep copy of t's exported fields. +func (t *http_Transport) Clone() *http_Transport { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + t2 := &http_Transport{ + Proxy: t.Proxy, + OnProxyConnectResponse: t.OnProxyConnectResponse, + DialContext: t.DialContext, + Dial: t.Dial, + DialTLS: t.DialTLS, + DialTLSContext: t.DialTLSContext, + TLSHandshakeTimeout: t.TLSHandshakeTimeout, + DisableKeepAlives: t.DisableKeepAlives, + DisableCompression: t.DisableCompression, + MaxIdleConns: t.MaxIdleConns, + MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, + MaxConnsPerHost: t.MaxConnsPerHost, + IdleConnTimeout: t.IdleConnTimeout, + ResponseHeaderTimeout: t.ResponseHeaderTimeout, + ExpectContinueTimeout: t.ExpectContinueTimeout, + ProxyConnectHeader: t.ProxyConnectHeader.Clone(), + GetProxyConnectHeader: t.GetProxyConnectHeader, + MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, + ForceAttemptHTTP2: t.ForceAttemptHTTP2, + WriteBufferSize: t.WriteBufferSize, + ReadBufferSize: t.ReadBufferSize, + } + if t.TLSClientConfig != nil { + t2.TLSClientConfig = t.TLSClientConfig.Clone() + } + if t.HTTP2 != nil { + t2.HTTP2 = &http_HTTP2Config{} + *t2.HTTP2 = *t.HTTP2 + } + if t.Protocols != nil { + t2.Protocols = &http_Protocols{} + *t2.Protocols = *t.Protocols + } + if !t.tlsNextProtoWasNil { + npm := maps.Clone(t.TLSNextProto) + if npm == nil { + npm = make(map[string]func(authority string, c *tls.Conn) http_RoundTripper) + } + t2.TLSNextProto = npm + } + return t2 +} + +type http_dialClientConner interface { + // DialClientConn creates a new client connection to address. + // + // If proxy is non-nil, the connection should use the provided proxy. + // If HTTP/3 proxies are not supported, DialClientConn should return + // an error wrapping [errors.ErrUnsupported]. + // + // The RoundTripper returned by DialClientConn must also implement the + // following methods to support [ClientConn] methods of the same name: + // Close() error + // Err() error + // Reserve() error + // Release() error + // Available() int + // InFlight() int + // + // The client connection should arrange to call internalStateHook + // when the connection closes, when requests complete, and when the + // connection concurrency limit changes. + // + // The client connection must call the internal state hook when + // the connection state changes asynchronously, such as when a request completes. + // + // The internal state hook need not be called after synchronous changes + // to the state: Close, Reserve, Release, and RoundTrip calls + // which don't start a request do not need to call the hook. + DialClientConn(ctx context.Context, address string, proxy *url.URL, internalStateHook func()) (http_RoundTripper, error) +} + +type http_closeIdleConnectionser interface { + // CloseIdleConnections is called by Transport.CloseIdleConnections. + // + // We expect to use this on transports supplied by x/net/http2 or x/net/http3. + // + // The transport will close idle connections created with DialClientConn + // before calling this method. The HTTP/3 transport should not attempt to + // close idle connections, but may clean up shared resources such as UDP + // sockets if no connections remain. + CloseIdleConnections() +} + +func (t *http_Transport) hasCustomTLSDialer() bool { + return t.DialTLS != nil || t.DialTLSContext != nil +} + +var http_http2client = godebug.New("http2client") + +// onceSetNextProtoDefaults initializes TLSNextProto. +// It must be called via t.nextProtoOnce.Do. +func (t *http_Transport) onceSetNextProtoDefaults() { + t.tlsNextProtoWasNil = (t.TLSNextProto == nil) + if http_http2client.Value() == "0" { + http_http2client.IncNonDefault() + return + } + + // If they've already configured http2 with + // golang.org/x/net/http2 instead of the bundled copy, try to + // get at its http2.Transport value (via the "https" + // altproto map) so we can call CloseIdleConnections on it if + // requested. (Issue 22891) + altProto, _ := t.altProto.Load().(map[string]http_RoundTripper) + if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { + if v := rv.Field(0); v.CanInterface() { + if h2i, ok := v.Interface().(http_closeIdleConnectionser); ok { + t.closeIdleFunc = h2i + return + } + } + } + + if _, ok := t.TLSNextProto["h2"]; ok { + // There's an existing HTTP/2 implementation installed. + return + } + protocols := t.protocols() + if !protocols.HTTP2() && !protocols.UnencryptedHTTP2() { + return + } + if http_omitBundledHTTP2 { + return + } + + t.configureHTTP2(protocols) +} + +func (t *http_Transport) protocols() http_Protocols { + if t.Protocols != nil { + return *t.Protocols // user-configured set + } + var p http_Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + switch { + case t.TLSNextProto != nil: + // Setting TLSNextProto to an empty map is a documented way + // to disable HTTP/2 on a Transport. + if t.TLSNextProto["h2"] != nil { + p.SetHTTP2(true) + } + case !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()): + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // Transport.Protocols.SetHTTP2(true) so we don't surprise them + // by modifying their tls.Config. Issue 14275. + // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + case http_http2client.Value() == "0": + default: + p.SetHTTP2(true) + } + return p } // ProxyFromEnvironment returns the URL of the proxy to use for a // given request, as indicated by the environment variables // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions -// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https -// requests. +// thereof, which take precedence over the uppercase versions). +// Requests use the proxy from the environment variable +// matching their scheme, unless excluded by NO_PROXY. // // The environment values may be either a complete URL or a // "host[:port]", in which case the "http" scheme is assumed. @@ -6455,60 +14642,103 @@ // // As a special case, if req.URL.Host is "localhost" (with or without // a port number), then a nil URL and nil error will be returned. -func ProxyFromEnvironment(req *Request) (*url.URL, error) { - var proxy string - if req.URL.Scheme == "https" { - proxy = httpsProxyEnv.Get() - } - if proxy == "" { - proxy = httpProxyEnv.Get() - } - if proxy == "" { - return nil, nil - } - if !useProxy(canonicalAddr(req.URL)) { - return nil, nil - } - proxyURL, err := url.Parse(proxy) - if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { - - if proxyURL, err := url.Parse("http://" + proxy); err == nil { - return proxyURL, nil - } - } - if err != nil { - return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) - } - return proxyURL, nil +func http_ProxyFromEnvironment(req *http_Request) (*url.URL, error) { + return http_envProxyFunc()(req.URL) } -// ProxyURL returns a proxy function (for use in a Transport) +// ProxyURL returns a proxy function (for use in a [Transport]) // that always returns the same URL. -func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { - return func(*Request) (*url.URL, error) { +func http_ProxyURL(fixedURL *url.URL) func(*http_Request) (*url.URL, error) { + return func(*http_Request) (*url.URL, error) { return fixedURL, nil } } // transportRequest is a wrapper around a *Request that adds -// optional extra headers to write. -type transportRequest struct { - *Request // original request, not to be mutated - extra Header // extra headers to write, or nil +// optional extra headers to write and stores any error to return +// from roundTrip. +type http_transportRequest struct { + *http_Request // original request, not to be mutated + extra http_Header // extra headers to write, or nil + trace *httptrace.ClientTrace // optional + + ctx context.Context // canceled when we are done with the request + cancel context.CancelCauseFunc + + mu sync.Mutex // guards err + err error // first setError value for mapRoundTripError to consider } -func (tr *transportRequest) extraHeaders() Header { +func (tr *http_transportRequest) extraHeaders() http_Header { if tr.extra == nil { - tr.extra = make(Header) + tr.extra = make(http_Header) } return tr.extra } -// RoundTrip implements the RoundTripper interface. -// -// For higher-level HTTP client support (such as handling of cookies -// and redirects), see Get, Post, and the Client type. -func (t *Transport) RoundTrip(req *Request) (resp *Response, err error) { +func (tr *http_transportRequest) setError(err error) { + tr.mu.Lock() + if tr.err == nil { + tr.err = err + } + tr.mu.Unlock() +} + +// useRegisteredProtocol reports whether an alternate protocol (as registered +// with Transport.RegisterProtocol) should be respected for this request. +func (t *http_Transport) useRegisteredProtocol(req *http_Request) bool { + if req.URL.Scheme == "https" && req.requiresHTTP1() { + // If this request requires HTTP/1, don't use the + // "https" alternate protocol, which is used by the + // HTTP/2 code to take over requests if there's an + // existing cached HTTP/2 connection. + return false + } + return true +} + +// alternateRoundTripper returns the alternate RoundTripper to use +// for this request if the Request's URL scheme requires one, +// or nil for the normal case of using the Transport. +func (t *http_Transport) alternateRoundTripper(req *http_Request) http_RoundTripper { + if !t.useRegisteredProtocol(req) { + return nil + } + if req.URL.Scheme == "https" && t.h2Config != nil && t.h2Config.ExternalRoundTrip() { + // This Transport has been configured to use an x/net/http2 Transport + // with a user-provided ClientConnPool. We're going to pass off the + // RoundTrip to x/net/http2 so it can use that pool. + // + // The ClientConnPool API is deprecated, but we're doing our best here + // to continue supporting any users who are using it. + return t.h2Config + } + altProto, _ := t.altProto.Load().(map[string]http_RoundTripper) + return altProto[req.URL.Scheme] +} + +func http_validateHeaders(hdrs http_Header) string { + for k, vv := range hdrs { + if !httpguts.ValidHeaderFieldName(k) { + return fmt.Sprintf("field name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + // Don't include the value in the error, + // because it may be sensitive. + return fmt.Sprintf("field value for %q", k) + } + } + } + return "" +} + +// roundTrip implements a RoundTripper over HTTP. +func (t *http_Transport) roundTrip(req *http_Request) (_ *http_Response, err error) { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + ctx := req.Context() + trace := httptrace.ContextClientTrace(ctx) + if req.URL == nil { req.closeBody() return nil, errors.New("http: nil Request.URL") @@ -6517,998 +14747,2602 @@ req.closeBody() return nil, errors.New("http: nil Request.Header") } - if req.URL.Scheme != "http" && req.URL.Scheme != "https" { - t.altMu.RLock() - var rt RoundTripper - if t.altProto != nil { - rt = t.altProto[req.URL.Scheme] - } - t.altMu.RUnlock() - if rt == nil { + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { + // Validate the outgoing headers. + if err := http_validateHeaders(req.Header); err != "" { req.closeBody() - return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + return nil, fmt.Errorf("net/http: invalid header %s", err) } - return rt.RoundTrip(req) + + // Validate the outgoing trailers too. + if err := http_validateHeaders(req.Trailer); err != "" { + req.closeBody() + return nil, fmt.Errorf("net/http: invalid trailer %s", err) + } + } + + origReq := req + req = http_setupRewindBody(req) + + if altRT := t.alternateRoundTripper(req); altRT != nil { + if resp, err := altRT.RoundTrip(req); err != http_ErrSkipAltProtocol { + return resp, err + } + var err error + req, err = http_rewindBody(req) + if err != nil { + return nil, err + } + } + if !isHTTP { + req.closeBody() + return nil, http_badStringError("unsupported protocol scheme", scheme) + } + if req.Method != "" && !http_validMethod(req.Method) { + req.closeBody() + return nil, fmt.Errorf("net/http: invalid method %q", req.Method) } if req.URL.Host == "" { req.closeBody() return nil, errors.New("http: no Host in request URL") } - treq := &transportRequest{Request: req} - cm, err := t.connectMethodForRequest(treq) - if err != nil { - req.closeBody() - return nil, err + + // Transport request context. + // + // If RoundTrip returns an error, it cancels this context before returning. + // + // If RoundTrip returns no error: + // - For an HTTP/1 request, persistConn.readLoop cancels this context + // after reading the request body. + // - For an HTTP/2 request, RoundTrip cancels this context after the HTTP/2 + // RoundTripper returns. + ctx, cancel := context.WithCancelCause(req.Context()) + + // Convert Request.Cancel into context cancellation. + if origReq.Cancel != nil { + go http_awaitLegacyCancel(ctx, cancel, origReq) } - pconn, err := t.getConn(req, cm) - if err != nil { - t.setReqCanceler(req, nil) - req.closeBody() - return nil, err - } + // Convert Transport.CancelRequest into context cancellation. + // + // This is lamentably expensive. CancelRequest has been deprecated for a long time + // and doesn't work on HTTP/2 requests. Perhaps we should drop support for it entirely. + cancel = t.prepareTransportCancel(origReq, cancel) - return pconn.roundTrip(treq) + defer func() { + if err != nil { + cancel(err) + } + }() + + for { + select { + case <-ctx.Done(): + req.closeBody() + return nil, context.Cause(ctx) + default: + } + + // treq gets modified by roundTrip, so we need to recreate for each retry. + treq := &http_transportRequest{http_Request: req, trace: trace, ctx: ctx, cancel: cancel} + cm, err := t.connectMethodForRequest(treq) + if err != nil { + req.closeBody() + return nil, err + } + + // Get the cached or newly-created connection to either the + // host (for http or https), the http proxy, or the http proxy + // pre-CONNECTed to https server. In any case, we'll be ready + // to send it requests. + pconn, err := t.getConn(treq, cm) + if err != nil { + req.closeBody() + return nil, err + } + + var resp *http_Response + if pconn.alt != nil { + // HTTP/2 path. + resp, err = pconn.alt.RoundTrip(req) + } else { + resp, err = pconn.roundTrip(treq) + } + if err == nil { + if pconn.alt != nil { + // HTTP/2 requests are not cancelable with CancelRequest, + // so we have no further need for the request context. + // + // On the HTTP/1 path, roundTrip takes responsibility for + // canceling the context after the response body is read. + cancel(http_errRequestDone) + } + resp.Request = origReq + return resp, nil + } + + // Failed. Clean up and determine whether to retry. + if http_http2isNoCachedConnError(err) { + if t.removeIdleConn(pconn) { + t.decConnsPerHost(pconn.cacheKey) + } + } else if !pconn.shouldRetryRequest(req, err) { + // Issue 16465: return underlying net.Conn.Read error from peek, + // as we've historically done. + if e, ok := err.(http_nothingWrittenError); ok { + err = e.error + } + if e, ok := err.(http_transportReadFromServerError); ok { + err = e.err + } + if b, ok := req.Body.(*http_readTrackingBody); ok && !b.didClose.Load() { + // Issue 49621: Close the request body if pconn.roundTrip + // didn't do so already. This can happen if the pconn + // write loop exits without reading the write request. + req.closeBody() + } + return nil, err + } + http_testHookRoundTripRetried() + + // Rewind the body if we're able to. + req, err = http_rewindBody(req) + if err != nil { + return nil, err + } + } } +func http_http2isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +func http_awaitLegacyCancel(ctx context.Context, cancel context.CancelCauseFunc, req *http_Request) { + select { + case <-req.Cancel: + cancel(http_errRequestCanceled) + case <-ctx.Done(): + } +} + +var http_errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") + +type http_readTrackingBody struct { + io.ReadCloser + didRead bool // not atomic.Bool because only one goroutine (the user's) should be accessing + didClose atomic.Bool +} + +func (r *http_readTrackingBody) Read(data []byte) (int, error) { + r.didRead = true + return r.ReadCloser.Read(data) +} + +func (r *http_readTrackingBody) Close() error { + if !r.didClose.CompareAndSwap(false, true) { + return nil + } + return r.ReadCloser.Close() +} + +// setupRewindBody returns a new request with a custom body wrapper +// that can report whether the body needs rewinding. +// This lets rewindBody avoid an error result when the request +// does not have GetBody but the body hasn't been read at all yet. +func http_setupRewindBody(req *http_Request) *http_Request { + if req.Body == nil || req.Body == http_NoBody { + return req + } + newReq := *req + newReq.Body = &http_readTrackingBody{ReadCloser: req.Body} + return &newReq +} + +// rewindBody returns a new request with the body rewound. +// It returns req unmodified if the body does not need rewinding. +// rewindBody takes care of closing req.Body when appropriate +// (in all cases except when rewindBody returns req unmodified). +func http_rewindBody(req *http_Request) (rewound *http_Request, err error) { + if req.Body == nil || req.Body == http_NoBody || (!req.Body.(*http_readTrackingBody).didRead && !req.Body.(*http_readTrackingBody).didClose.Load()) { + return req, nil // nothing to rewind + } + if !req.Body.(*http_readTrackingBody).didClose.Load() { + req.closeBody() + } + if req.GetBody == nil { + return nil, http_errCannotRewind + } + body, err := req.GetBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = &http_readTrackingBody{ReadCloser: body} + return &newReq, nil +} + +// shouldRetryRequest reports whether we should retry sending a failed +// HTTP request on a new connection. The non-nil input error is the +// error from roundTrip. +func (pc *http_persistConn) shouldRetryRequest(req *http_Request, err error) bool { + if http_http2isNoCachedConnError(err) { + // Issue 16582: if the user started a bunch of + // requests at once, they can all pick the same conn + // and violate the server's max concurrent streams. + // Instead, match the HTTP/1 behavior for now and dial + // again to get a new TCP connection, rather than failing + // this request. + return true + } + if err == http_errMissingHost { + // User error. + return false + } + if !pc.isReused() { + // This was a fresh connection. There's no reason the server + // should've hung up on us. + // + // Also, if we retried now, we could loop forever + // creating new connections and retrying if the server + // is just hanging up on us because it doesn't like + // our request (as opposed to sending an error). + return false + } + if _, ok := err.(http_nothingWrittenError); ok { + // We never wrote anything, so it's safe to retry, if there's no body or we + // can "rewind" the body with GetBody. + return req.outgoingLength() == 0 || req.GetBody != nil + } + if !req.isReplayable() { + // Don't retry non-idempotent requests. + return false + } + if _, ok := err.(http_transportReadFromServerError); ok { + // We got some non-EOF net.Conn.Read failure reading + // the 1st response byte from the server. + return true + } + if err == http_errServerClosedIdle { + // The server replied with io.EOF while we were trying to + // read the response. Probably an unfortunately keep-alive + // timeout, just as the client was writing a request. + return true + } + return false // conservatively +} + +// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. +var http_ErrSkipAltProtocol = internal.ErrSkipAltProtocol + // RegisterProtocol registers a new protocol with scheme. -// The Transport will pass requests using the given scheme to rt. +// The [Transport] will pass requests using the given scheme to rt. // It is rt's responsibility to simulate HTTP request semantics. // // RegisterProtocol can be used by other packages to provide // implementations of protocol schemes like "ftp" or "file". -func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { - if scheme == "http" || scheme == "https" { - panic("protocol " + scheme + " already registered") +// +// If rt.RoundTrip returns [ErrSkipAltProtocol], the Transport will +// handle the [Transport.RoundTrip] itself for that one request, as if the +// protocol were not registered. +func (t *http_Transport) RegisterProtocol(scheme string, rt http_RoundTripper) { + if err := t.registerProtocol(scheme, rt); err != nil { + panic(err) } +} + +func (t *http_Transport) registerProtocol(scheme string, rt http_RoundTripper) error { t.altMu.Lock() defer t.altMu.Unlock() - if t.altProto == nil { - t.altProto = make(map[string]RoundTripper) + + if scheme == "http/2" { + if t.h2Config != nil { + panic("http: HTTP/2 Transport already registered") + } + var ok bool + if t.h2Config, ok = rt.(http_http2ExternalTransportConfig); !ok { + panic("http: HTTP/2 configuration does not implement ExternalTransportConfig") + } + t.h2Config.Registered(t) } - if _, exists := t.altProto[scheme]; exists { - panic("protocol " + scheme + " already registered") + + if scheme == "http/3" { + var ok bool + if t.h3Transport, ok = rt.(http_dialClientConner); !ok { + panic("http: HTTP/3 RoundTripper does not implement DialClientConn") + } } - t.altProto[scheme] = rt + + oldMap, _ := t.altProto.Load().(map[string]http_RoundTripper) + if _, exists := oldMap[scheme]; exists { + return errors.New("protocol " + scheme + " already registered") + } + newMap := maps.Clone(oldMap) + if newMap == nil { + newMap = make(map[string]http_RoundTripper) + } + newMap[scheme] = rt + t.altProto.Store(newMap) + return nil } // CloseIdleConnections closes any connections which were previously // connected from previous requests but are now sitting idle in // a "keep-alive" state. It does not interrupt any connections currently // in use. -func (t *Transport) CloseIdleConnections() { +func (t *http_Transport) CloseIdleConnections() { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) t.idleMu.Lock() m := t.idleConn t.idleConn = nil - t.idleConnCh = nil - t.wantIdle = true + t.closeIdle = true // close newly idle connections + t.idleLRU = http_connLRU{} t.idleMu.Unlock() for _, conns := range m { for _, pconn := range conns { - pconn.close() + pconn.close(http_errCloseIdleConns) } } + t.connsPerHostMu.Lock() + t.dialsInProgress.all(func(w *http_wantConn) { + if w.cancelCtx != nil && !w.waiting() { + w.cancelCtx() + } + }) + t.connsPerHostMu.Unlock() + + // Tell various associated transports to close their connections. + + // net/http/internal/http2 transport. This is the common case for HTTP/2 users. + if tr2 := t.h2Transport; tr2 != nil { + tr2.CloseIdleConnections() + } + // Probably an older x/net/http2 transport registered via Transport.RegisterProtocol. + // This is a legacy path; modern users just use internal/http2. + // (Note that we don't use this path when x/net/http2 wraps the net/http transport; + // this is supporting pre-wrapping x/net/http2.) + if t2 := t.closeIdleFunc; t2 != nil { + t2.CloseIdleConnections() + } + // HTTP/3 transport, probably from x/net/http3. + if cc, ok := t.h3Transport.(http_closeIdleConnectionser); ok { + cc.CloseIdleConnections() + } +} + +// prepareTransportCancel sets up state to convert Transport.CancelRequest into context cancellation. +func (t *http_Transport) prepareTransportCancel(req *http_Request, origCancel context.CancelCauseFunc) context.CancelCauseFunc { + // Historically, RoundTrip has not modified the Request in any way. + // We could avoid the need to keep a map of all in-flight requests by adding + // a field to the Request containing its cancel func, and setting that field + // while the request is in-flight. Callers aren't supposed to reuse a Request + // until after the response body is closed, so this wouldn't violate any + // concurrency guarantees. + cancel := func(err error) { + origCancel(err) + t.reqMu.Lock() + delete(t.reqCanceler, req) + t.reqMu.Unlock() + } + t.reqMu.Lock() + if t.reqCanceler == nil { + t.reqCanceler = make(map[*http_Request]context.CancelCauseFunc) + } + t.reqCanceler[req] = cancel + t.reqMu.Unlock() + return cancel } // CancelRequest cancels an in-flight request by closing its connection. -// CancelRequest should only be called after RoundTrip has returned. -func (t *Transport) CancelRequest(req *Request) { +// CancelRequest should only be called after [Transport.RoundTrip] has returned. +// +// Deprecated: Use [Request.WithContext] to create a request with a +// cancelable context instead. CancelRequest cannot cancel HTTP/2 +// requests. This may become a no-op in a future release of Go. +func (t *http_Transport) CancelRequest(req *http_Request) { t.reqMu.Lock() cancel := t.reqCanceler[req] - delete(t.reqCanceler, req) t.reqMu.Unlock() if cancel != nil { - cancel() + cancel(http_errRequestCanceled) } } +// +// Private implementation past this point. +// + var ( - httpProxyEnv = &envOnce{ - names: []string{"HTTP_PROXY", "http_proxy"}, - } - httpsProxyEnv = &envOnce{ - names: []string{"HTTPS_PROXY", "https_proxy"}, - } - noProxyEnv = &envOnce{ - names: []string{"NO_PROXY", "no_proxy"}, - } + http_envProxyOnce sync.Once + http_envProxyFuncValue func(*url.URL) (*url.URL, error) ) -// envOnce looks up an environment variable (optionally by multiple -// names) once. It mitigates expensive lookups on some platforms -// (e.g. Windows). -type envOnce struct { - names []string - once sync.Once - val string +// envProxyFunc returns a function that reads the +// environment variable to determine the proxy address. +func http_envProxyFunc() func(*url.URL) (*url.URL, error) { + http_envProxyOnce.Do(func() { + http_envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() + }) + return http_envProxyFuncValue } -func (e *envOnce) Get() string { - e.once.Do(e.init) - return e.val +// resetProxyConfig is used by tests. +func http_resetProxyConfig() { + http_envProxyOnce = sync.Once{} + http_envProxyFuncValue = nil } -func (e *envOnce) init() { - for _, n := range e.names { - e.val = os.Getenv(n) - if e.val != "" { - return - } - } -} - -// reset is used by tests -func (e *envOnce) reset() { - e.once = sync.Once{} - e.val = "" -} - -func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { +func (t *http_Transport) connectMethodForRequest(treq *http_transportRequest) (cm http_connectMethod, err error) { cm.targetScheme = treq.URL.Scheme - cm.targetAddr = canonicalAddr(treq.URL) + cm.targetAddr = http_canonicalAddr(treq.URL) if t.Proxy != nil { - cm.proxyURL, err = t.Proxy(treq.Request) + cm.proxyURL, err = t.Proxy(treq.http_Request) } + cm.onlyH1 = treq.requiresHTTP1() return cm, err } // proxyAuth returns the Proxy-Authorization header to set // on requests, if applicable. -func (cm *connectMethod) proxyAuth() string { +func (cm *http_connectMethod) proxyAuth() string { if cm.proxyURL == nil { return "" } if u := cm.proxyURL.User; u != nil { username := u.Username() password, _ := u.Password() - return "Basic " + basicAuth(username, password) + return "Basic " + http_basicAuth(username, password) } return "" } -// putIdleConn adds pconn to the list of idle persistent connections awaiting +// error values for debugging and testing, not seen by users. +var ( + http_errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") + http_errConnBroken = errors.New("http: putIdleConn: connection is in bad state") + http_errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") + http_errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") + http_errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") + http_errCloseIdleConns = errors.New("http: CloseIdleConnections called") + http_errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") + http_errIdleConnTimeout = errors.New("http: idle connection timeout") + + // errServerClosedIdle is not seen by users for idempotent requests, but may be + // seen by a user if the server shuts down an idle connection and sends its FIN + // in flight with already-written POST body bytes from the client. + // See https://github.com/golang/go/issues/19943#issuecomment-355607646 + http_errServerClosedIdle = errors.New("http: server closed idle connection") +) + +// transportReadFromServerError is used by Transport.readLoop when the +// 1 byte peek read fails and we're actually anticipating a response. +// Usually this is just due to the inherent keep-alive shut down race, +// where the server closed the connection at the same time the client +// wrote. The underlying err field is usually io.EOF or some +// ECONNRESET sort of thing which varies by platform. But it might be +// the user's custom net.Conn.Read error too, so we carry it along for +// them to return from Transport.RoundTrip. +type http_transportReadFromServerError struct { + err error +} + +func (e http_transportReadFromServerError) Unwrap() error { return e.err } + +func (e http_transportReadFromServerError) Error() string { + return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) +} + +func (t *http_Transport) putOrCloseIdleConn(pconn *http_persistConn) { + if err := t.tryPutIdleConn(pconn); err != nil { + pconn.close(err) + } +} + +func (t *http_Transport) maxIdleConnsPerHost() int { + if v := t.MaxIdleConnsPerHost; v != 0 { + return v + } + return http_DefaultMaxIdleConnsPerHost +} + +// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting // a new request. -// If pconn is no longer needed or not in a good state, putIdleConn -// returns false. -func (t *Transport) putIdleConn(pconn *persistConn) bool { +// If pconn is no longer needed or not in a good state, tryPutIdleConn returns +// an error explaining why it wasn't registered. +// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. +func (t *http_Transport) tryPutIdleConn(pconn *http_persistConn) error { if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { - pconn.close() - return false + return http_errKeepAlivesDisabled } if pconn.isBroken() { - return false + return http_errConnBroken } - key := pconn.cacheKey - max := t.MaxIdleConnsPerHost - if max == 0 { - max = DefaultMaxIdleConnsPerHost + pconn.markReused() + if pconn.isClientConn { + // internalStateHook is always set for conns created by NewClientConn. + defer pconn.internalStateHook() + pconn.mu.Lock() + defer pconn.mu.Unlock() + if !pconn.inFlight { + panic("pconn is not in flight") + } + pconn.inFlight = false + select { + case pconn.availch <- struct{}{}: + default: + panic("unable to make pconn available") + } + return nil } + t.idleMu.Lock() + defer t.idleMu.Unlock() - waitingDialer := t.idleConnCh[key] - select { - case waitingDialer <- pconn: + // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list, + // because multiple goroutines can use them simultaneously. + // If this is an HTTP/2 connection being “returned,” we're done. + if pconn.alt != nil && t.idleLRU.m[pconn] != nil { + return nil + } - t.idleMu.Unlock() - return true - default: - if waitingDialer != nil { - - delete(t.idleConnCh, key) + // Deliver pconn to goroutine waiting for idle connection, if any. + // (They may be actively dialing, but this conn is ready first. + // Chrome calls this socket late binding. + // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.) + key := pconn.cacheKey + if q, ok := t.idleConnWait[key]; ok { + done := false + if pconn.alt == nil { + // HTTP/1. + // Loop over the waiting list until we find a w that isn't done already, and hand it pconn. + for q.len() > 0 { + w := q.popFront() + if w.tryDeliver(pconn, nil, time.Time{}) { + done = true + break + } + } + } else { + // HTTP/2. + // Can hand the same pconn to everyone in the waiting list, + // and we still won't be done: we want to put it in the idle + // list unconditionally, for any future clients too. + for q.len() > 0 { + w := q.popFront() + w.tryDeliver(pconn, nil, time.Time{}) + } + } + if q.len() == 0 { + delete(t.idleConnWait, key) + } else { + t.idleConnWait[key] = q + } + if done { + return nil } } - if t.wantIdle { - t.idleMu.Unlock() - pconn.close() - return false + + if t.closeIdle { + return http_errCloseIdle } if t.idleConn == nil { - t.idleConn = make(map[connectMethodKey][]*persistConn) + t.idleConn = make(map[http_connectMethodKey][]*http_persistConn) } - if len(t.idleConn[key]) >= max { - t.idleMu.Unlock() - pconn.close() - return false + idles := t.idleConn[key] + if len(idles) >= t.maxIdleConnsPerHost() { + return http_errTooManyIdleHost } - for _, exist := range t.idleConn[key] { + for _, exist := range idles { if exist == pconn { log.Fatalf("dup idle pconn %p in freelist", pconn) } } - t.idleConn[key] = append(t.idleConn[key], pconn) - t.idleMu.Unlock() + t.idleConn[key] = append(idles, pconn) + t.idleLRU.add(pconn) + if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { + oldest := t.idleLRU.removeOldest() + oldest.close(http_errTooManyIdle) + t.removeIdleConnLocked(oldest) + } + + // Set idle timer, but only for HTTP/1 (pconn.alt == nil). + // The HTTP/2 implementation manages the idle timer itself + // (see idleConnTimeout in h2_bundle.go). + if t.IdleConnTimeout > 0 && pconn.alt == nil { + if pconn.idleTimer != nil { + pconn.idleTimer.Reset(t.IdleConnTimeout) + } else { + pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + } + } + pconn.idleAt = time.Now() + return nil +} + +// queueForIdleConn queues w to receive the next idle connection for w.cm. +// As an optimization hint to the caller, queueForIdleConn reports whether +// it successfully delivered an already-idle connection. +func (t *http_Transport) queueForIdleConn(w *http_wantConn) (delivered bool) { + if t.DisableKeepAlives { + return false + } + + t.idleMu.Lock() + defer t.idleMu.Unlock() + + // Stop closing connections that become idle - we might want one. + // (That is, undo the effect of t.CloseIdleConnections.) + t.closeIdle = false + + if w == nil { + // Happens in test hook. + return false + } + + // If IdleConnTimeout is set, calculate the oldest + // persistConn.idleAt time we're willing to use a cached idle + // conn. + var oldTime time.Time + if t.IdleConnTimeout > 0 { + oldTime = time.Now().Add(-t.IdleConnTimeout) + } + + // Look for most recently-used idle connection. + if list, ok := t.idleConn[w.key]; ok { + stop := false + delivered := false + for len(list) > 0 && !stop { + pconn := list[len(list)-1] + + // See whether this connection has been idle too long, considering + // only the wall time (the Round(0)), in case this is a laptop or VM + // coming out of suspend with previously cached idle connections. + tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) + if tooOld { + // Async cleanup. Launch in its own goroutine (as if a + // time.AfterFunc called it); it acquires idleMu, which we're + // holding, and does a synchronous net.Conn.Close. + go pconn.closeConnIfStillIdle() + } + if pconn.isBroken() || tooOld { + // If either persistConn.readLoop has marked the connection + // broken, but Transport.removeIdleConn has not yet removed it + // from the idle list, or if this persistConn is too old (it was + // idle too long), then ignore it and look for another. In both + // cases it's already in the process of being closed. + list = list[:len(list)-1] + continue + } + delivered = w.tryDeliver(pconn, nil, pconn.idleAt) + if delivered { + if pconn.alt != nil { + // HTTP/2: multiple clients can share pconn. + // Leave it in the list. + } else { + // HTTP/1: only one client can use pconn. + // Remove it from the list. + t.idleLRU.remove(pconn) + list = list[:len(list)-1] + } + } + stop = true + } + if len(list) > 0 { + t.idleConn[w.key] = list + } else { + delete(t.idleConn, w.key) + } + if stop { + return delivered + } + } + + // Register to receive next connection that becomes idle. + if t.idleConnWait == nil { + t.idleConnWait = make(map[http_connectMethodKey]http_wantConnQueue) + } + q := t.idleConnWait[w.key] + q.cleanFrontNotWaiting() + q.pushBack(w) + t.idleConnWait[w.key] = q + return false +} + +// removeIdleConn marks pconn as dead. +func (t *http_Transport) removeIdleConn(pconn *http_persistConn) bool { + if pconn.isClientConn { + return true + } + t.idleMu.Lock() + defer t.idleMu.Unlock() + return t.removeIdleConnLocked(pconn) +} + +// t.idleMu must be held. +func (t *http_Transport) removeIdleConnLocked(pconn *http_persistConn) bool { + if pconn.idleTimer != nil { + pconn.idleTimer.Stop() + } + t.idleLRU.remove(pconn) + key := pconn.cacheKey + pconns := t.idleConn[key] + var removed bool + switch len(pconns) { + case 0: + // Nothing + case 1: + if pconns[0] == pconn { + delete(t.idleConn, key) + removed = true + } + default: + for i, v := range pconns { + if v != pconn { + continue + } + // Slide down, keeping most recently-used + // conns at the end. + copy(pconns[i:], pconns[i+1:]) + t.idleConn[key] = pconns[:len(pconns)-1] + removed = true + break + } + } + return removed +} + +var http_zeroDialer net.Dialer + +func (t *http_Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { + if t.DialContext != nil { + c, err := t.DialContext(ctx, network, addr) + if c == nil && err == nil { + err = errors.New("net/http: Transport.DialContext hook returned (nil, nil)") + } + return c, err + } + if t.Dial != nil { + c, err := t.Dial(network, addr) + if c == nil && err == nil { + err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") + } + return c, err + } + return http_zeroDialer.DialContext(ctx, network, addr) +} + +// A wantConn records state about a wanted connection +// (that is, an active call to getConn). +// The conn may be gotten by dialing or by finding an idle connection, +// or a cancellation may make the conn no longer wanted. +// These three options are racing against each other and use +// wantConn to coordinate and agree about the winning outcome. +type http_wantConn struct { + cm http_connectMethod + key http_connectMethodKey // cm.key() + + // hooks for testing to know when dials are done + // beforeDial is called in the getConn goroutine when the dial is queued. + // afterDial is called when the dial is completed or canceled. + beforeDial func() + afterDial func() + + mu sync.Mutex // protects ctx, done and sending of the result + ctx context.Context // context for dial, cleared after delivered or canceled + cancelCtx context.CancelFunc + done bool // true after delivered or canceled + result chan http_connOrError // channel to deliver connection or error +} + +type http_connOrError struct { + pc *http_persistConn + err error + idleAt time.Time +} + +// waiting reports whether w is still waiting for an answer (connection or error). +func (w *http_wantConn) waiting() bool { + w.mu.Lock() + defer w.mu.Unlock() + + return !w.done +} + +// getCtxForDial returns context for dial or nil if connection was delivered or canceled. +func (w *http_wantConn) getCtxForDial() context.Context { + w.mu.Lock() + defer w.mu.Unlock() + + return w.ctx +} + +// tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. +func (w *http_wantConn) tryDeliver(pc *http_persistConn, err error, idleAt time.Time) bool { + w.mu.Lock() + defer w.mu.Unlock() + + if w.done { + return false + } + if (pc == nil) == (err == nil) { + panic("net/http: internal error: misuse of tryDeliver") + } + w.ctx = nil + w.done = true + + w.result <- http_connOrError{pc: pc, err: err, idleAt: idleAt} + close(w.result) + return true } -// getIdleConnCh returns a channel to receive and return idle -// persistent connection for the given connectMethod. -// It may return nil, if persistent connections are not being used. -func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { - if t.DisableKeepAlives { - return nil +// cancel marks w as no longer wanting a result (for example, due to cancellation). +// If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. +func (w *http_wantConn) cancel(t *http_Transport) { + w.mu.Lock() + var pc *http_persistConn + if w.done { + if r, ok := <-w.result; ok { + pc = r.pc + } + } else { + close(w.result) } - key := cm.key() - t.idleMu.Lock() - defer t.idleMu.Unlock() - t.wantIdle = false - if t.idleConnCh == nil { - t.idleConnCh = make(map[connectMethodKey]chan *persistConn) + w.ctx = nil + w.done = true + w.mu.Unlock() + + // HTTP/2 connections (pc.alt != nil) aren't removed from the idle pool on use, + // and should not be added back here. If the pconn isn't in the idle pool, + // it's because we removed it due to an error. + if pc != nil && pc.alt == nil { + t.putOrCloseIdleConn(pc) } - ch, ok := t.idleConnCh[key] - if !ok { - ch = make(chan *persistConn) - t.idleConnCh[key] = ch - } - return ch } -func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { - key := cm.key() - t.idleMu.Lock() - defer t.idleMu.Unlock() - if t.idleConn == nil { - return nil - } - for { - pconns, ok := t.idleConn[key] - if !ok { +// A wantConnQueue is a queue of wantConns. +type http_wantConnQueue struct { + // This is a queue, not a deque. + // It is split into two stages - head[headPos:] and tail. + // popFront is trivial (headPos++) on the first stage, and + // pushBack is trivial (append) on the second stage. + // If the first stage is empty, popFront can swap the + // first and second stages to remedy the situation. + // + // This two-stage split is analogous to the use of two lists + // in Okasaki's purely functional queue but without the + // overhead of reversing the list when swapping stages. + head []*http_wantConn + headPos int + tail []*http_wantConn +} + +// len returns the number of items in the queue. +func (q *http_wantConnQueue) len() int { + return len(q.head) - q.headPos + len(q.tail) +} + +// pushBack adds w to the back of the queue. +func (q *http_wantConnQueue) pushBack(w *http_wantConn) { + q.tail = append(q.tail, w) +} + +// popFront removes and returns the wantConn at the front of the queue. +func (q *http_wantConnQueue) popFront() *http_wantConn { + if q.headPos >= len(q.head) { + if len(q.tail) == 0 { return nil } - if len(pconns) == 1 { - pconn = pconns[0] - delete(t.idleConn, key) - } else { + // Pick up tail as new head, clear tail. + q.head, q.headPos, q.tail = q.tail, 0, q.head[:0] + } + w := q.head[q.headPos] + q.head[q.headPos] = nil + q.headPos++ + return w +} - pconn = pconns[len(pconns)-1] - t.idleConn[key] = pconns[:len(pconns)-1] +// peekFront returns the wantConn at the front of the queue without removing it. +func (q *http_wantConnQueue) peekFront() *http_wantConn { + if q.headPos < len(q.head) { + return q.head[q.headPos] + } + if len(q.tail) > 0 { + return q.tail[0] + } + return nil +} + +// cleanFrontNotWaiting pops any wantConns that are no longer waiting from the head of the +// queue, reporting whether any were popped. +func (q *http_wantConnQueue) cleanFrontNotWaiting() (cleaned bool) { + for { + w := q.peekFront() + if w == nil || w.waiting() { + return cleaned } - if !pconn.isBroken() { + q.popFront() + cleaned = true + } +} + +// cleanFrontCanceled pops any wantConns with canceled dials from the head of the queue. +func (q *http_wantConnQueue) cleanFrontCanceled() { + for { + w := q.peekFront() + if w == nil || w.cancelCtx != nil { + return + } + q.popFront() + } +} + +// all iterates over all wantConns in the queue. +// The caller must not modify the queue while iterating. +func (q *http_wantConnQueue) all(f func(*http_wantConn)) { + for _, w := range q.head[q.headPos:] { + f(w) + } + for _, w := range q.tail { + f(w) + } +} + +func (t *http_Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { + if t.DialTLSContext != nil { + conn, err = t.DialTLSContext(ctx, network, addr) + } else { + conn, err = t.DialTLS(network, addr) + } + if conn == nil && err == nil { + err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") + } + return +} + +// getConn dials and creates a new persistConn to the target as +// specified in the connectMethod. This includes doing a proxy CONNECT +// and/or setting up TLS. If this doesn't return an error, the persistConn +// is ready to write requests to. +func (t *http_Transport) getConn(treq *http_transportRequest, cm http_connectMethod) (_ *http_persistConn, err error) { + req := treq.http_Request + trace := treq.trace + ctx := req.Context() + if trace != nil && trace.GetConn != nil { + trace.GetConn(cm.addr()) + } + + // Detach from the request context's cancellation signal. + // The dial should proceed even if the request is canceled, + // because a future request may be able to make use of the connection. + // + // We retain the request context's values. + dialCtx, dialCancel := context.WithCancel(context.WithoutCancel(ctx)) + + w := &http_wantConn{ + cm: cm, + key: cm.key(), + ctx: dialCtx, + cancelCtx: dialCancel, + result: make(chan http_connOrError, 1), + beforeDial: http_testHookPrePendingDial, + afterDial: http_testHookPostPendingDial, + } + defer func() { + if err != nil { + w.cancel(t) + } + }() + + // Queue for idle connection. + if delivered := t.queueForIdleConn(w); !delivered { + t.queueForDial(w) + } + + // Wait for completion or cancellation. + select { + case r := <-w.result: + // Trace success but only for HTTP/1. + // HTTP/2 calls trace.GotConn itself. + if r.pc != nil && r.pc.alt == nil && trace != nil && trace.GotConn != nil { + info := httptrace.GotConnInfo{ + Conn: r.pc.conn, + Reused: r.pc.isReused(), + } + if !r.idleAt.IsZero() { + info.WasIdle = true + info.IdleTime = time.Since(r.idleAt) + } + trace.GotConn(info) + } + if r.err != nil { + // If the request has been canceled, that's probably + // what caused r.err; if so, prefer to return the + // cancellation error (see golang.org/issue/16049). + select { + case <-treq.ctx.Done(): + err := context.Cause(treq.ctx) + if err == http_errRequestCanceled { + err = http_errRequestCanceledConn + } + return nil, err + default: + // return below + } + } + return r.pc, r.err + case <-treq.ctx.Done(): + err := context.Cause(treq.ctx) + if err == http_errRequestCanceled { + err = http_errRequestCanceledConn + } + return nil, err + } +} + +// queueForDial queues w to wait for permission to begin dialing. +// Once w receives permission to dial, it will do so in a separate goroutine. +func (t *http_Transport) queueForDial(w *http_wantConn) { + w.beforeDial() + + t.connsPerHostMu.Lock() + defer t.connsPerHostMu.Unlock() + + if t.MaxConnsPerHost <= 0 { + t.startDialConnForLocked(w) + return + } + + if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost { + if t.connsPerHost == nil { + t.connsPerHost = make(map[http_connectMethodKey]int) + } + t.connsPerHost[w.key] = n + 1 + t.startDialConnForLocked(w) + return + } + + if t.connsPerHostWait == nil { + t.connsPerHostWait = make(map[http_connectMethodKey]http_wantConnQueue) + } + q := t.connsPerHostWait[w.key] + q.cleanFrontNotWaiting() + q.pushBack(w) + t.connsPerHostWait[w.key] = q +} + +// startDialConnFor calls dialConn in a new goroutine. +// t.connsPerHostMu must be held. +func (t *http_Transport) startDialConnForLocked(w *http_wantConn) { + t.dialsInProgress.cleanFrontCanceled() + t.dialsInProgress.pushBack(w) + go func() { + t.dialConnFor(w) + t.connsPerHostMu.Lock() + defer t.connsPerHostMu.Unlock() + w.cancelCtx = nil + }() +} + +// dialConnFor dials on behalf of w and delivers the result to w. +// dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()]. +// If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()]. +func (t *http_Transport) dialConnFor(w *http_wantConn) { + defer w.afterDial() + ctx := w.getCtxForDial() + if ctx == nil { + t.decConnsPerHost(w.key) + return + } + + const isClientConn = false + pc, err := t.dialConn(ctx, w.cm, isClientConn, nil) + delivered := w.tryDeliver(pc, err, time.Time{}) + if err == nil && (!delivered || pc.alt != nil) { + // pconn was not passed to w, + // or it is HTTP/2 and can be shared. + // Add to the idle connection pool. + t.putOrCloseIdleConn(pc) + } + if err != nil { + t.decConnsPerHost(w.key) + } +} + +// decConnsPerHost decrements the per-host connection count for key, +// which may in turn give a different waiting goroutine permission to dial. +func (t *http_Transport) decConnsPerHost(key http_connectMethodKey) { + if t.MaxConnsPerHost <= 0 { + return + } + + t.connsPerHostMu.Lock() + defer t.connsPerHostMu.Unlock() + n := t.connsPerHost[key] + if n == 0 { + // Shouldn't happen, but if it does, the counting is buggy and could + // easily lead to a silent deadlock, so report the problem loudly. + panic("net/http: internal error: connCount underflow") + } + + // Can we hand this count to a goroutine still waiting to dial? + // (Some goroutines on the wait list may have timed out or + // gotten a connection another way. If they're all gone, + // we don't want to kick off any spurious dial operations.) + if q := t.connsPerHostWait[key]; q.len() > 0 { + done := false + for q.len() > 0 { + w := q.popFront() + if w.waiting() { + t.startDialConnForLocked(w) + done = true + break + } + } + if q.len() == 0 { + delete(t.connsPerHostWait, key) + } else { + // q is a value (like a slice), so we have to store + // the updated q back into the map. + t.connsPerHostWait[key] = q + } + if done { return } } -} -func (t *Transport) setReqCanceler(r *Request, fn func()) { - t.reqMu.Lock() - defer t.reqMu.Unlock() - if t.reqCanceler == nil { - t.reqCanceler = make(map[*Request]func()) - } - if fn != nil { - t.reqCanceler[r] = fn + // Otherwise, decrement the recorded count. + if n--; n == 0 { + delete(t.connsPerHost, key) } else { - delete(t.reqCanceler, r) + t.connsPerHost[key] = n } } -// replaceReqCanceler replaces an existing cancel function. If there is no cancel function -// for the request, we don't set the function and return false. -// Since CancelRequest will clear the canceler, we can use the return value to detect if -// the request was canceled since the last setReqCancel call. -func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { - t.reqMu.Lock() - defer t.reqMu.Unlock() - _, ok := t.reqCanceler[r] - if !ok { - return false +// Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS +// tunnel, this function establishes a nested TLS session inside the encrypted channel. +// The remote endpoint's name may be overridden by TLSClientConfig.ServerName. +func (pconn *http_persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error { + // Initiate TLS and check remote host name against certificate. + cfg := http_cloneTLSConfig(pconn.t.TLSClientConfig) + if cfg.ServerName == "" { + cfg.ServerName = name } - if fn != nil { - t.reqCanceler[r] = fn - } else { - delete(t.reqCanceler, r) + if pconn.cacheKey.onlyH1 { + cfg.NextProtos = nil } - return true -} - -func (t *Transport) dial(network, addr string) (c net.Conn, err error) { - if t.Dial != nil { - return t.Dial(network, addr) + plainConn := pconn.conn + tlsConn := tls.Client(plainConn, cfg) + errc := make(chan error, 2) + var timer *time.Timer // for canceling TLS handshake + if d := pconn.t.TLSHandshakeTimeout; d != 0 { + timer = time.AfterFunc(d, func() { + errc <- http_tlsHandshakeTimeoutError{} + }) } - return net.Dial(network, addr) -} - -// Testing hooks: -var prePendingDial, postPendingDial func() - -// getConn dials and creates a new persistConn to the target as -// specified in the connectMethod. This includes doing a proxy CONNECT -// and/or setting up TLS. If this doesn't return an error, the persistConn -// is ready to write requests to. -func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { - if pc := t.getIdleConn(cm); pc != nil { - - t.setReqCanceler(req, func() {}) - return pc, nil - } - - type dialRes struct { - pc *persistConn - err error - } - dialc := make(chan dialRes) - - prePendingDial := prePendingDial - postPendingDial := postPendingDial - - handlePendingDial := func() { - if prePendingDial != nil { - prePendingDial() - } - go func() { - if v := <-dialc; v.err == nil { - t.putIdleConn(v.pc) - } - if postPendingDial != nil { - postPendingDial() - } - }() - } - - cancelc := make(chan struct{}) - t.setReqCanceler(req, func() { close(cancelc) }) - go func() { - pc, err := t.dialConn(cm) - dialc <- dialRes{pc, err} + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + err := tlsConn.HandshakeContext(ctx) + if timer != nil { + timer.Stop() + } + errc <- err }() - - idleConnCh := t.getIdleConnCh(cm) - select { - case v := <-dialc: - - return v.pc, v.err - case pc := <-idleConnCh: - - handlePendingDial() - return pc, nil - case <-req.Cancel: - handlePendingDial() - return nil, errors.New("net/http: request canceled while waiting for connection") - case <-cancelc: - handlePendingDial() - return nil, errors.New("net/http: request canceled while waiting for connection") + if err := <-errc; err != nil { + plainConn.Close() + if err == (http_tlsHandshakeTimeoutError{}) { + // Now that we have closed the connection, + // wait for the call to HandshakeContext to return. + <-errc + } + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } + return err } + cs := tlsConn.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } + pconn.tlsState = &cs + pconn.conn = tlsConn + return nil } -func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { - pconn := &persistConn{ - t: t, - cacheKey: cm.key(), - reqch: make(chan requestAndChan, 1), - writech: make(chan writeRequest, 1), - closech: make(chan struct{}), - writeErrCh: make(chan error, 1), - } - tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil - if tlsDial { - var err error - pconn.conn, err = t.DialTLS("tcp", cm.addr()) +type http_erringRoundTripper interface { + RoundTripErr() error +} + +var http_testHookProxyConnectTimeout = context.WithTimeout + +func (t *http_Transport) dialConn(ctx context.Context, cm http_connectMethod, isClientConn bool, internalStateHook func()) (pconn *http_persistConn, err error) { + // TODO: actually support HTTP/3. Among other things: + // - make HTTP/3 play well with proxy. + // - implement happy eyeball between HTTP/3 and HTTP/1 & HTTP/2. + // - clean up the connection pooling logic. + if p := t.protocols(); p.http3() { + if p.HTTP1() || p.HTTP2() || p.UnencryptedHTTP2() { + return nil, errors.New("http: when using HTTP3, Transport.Protocols must contain only HTTP3") + } + if t.h3Transport == nil { + return nil, errors.New("http: Transport.Protocols contains HTTP3, but Transport does not support HTTP/3") + } + rt, err := t.h3Transport.DialClientConn(ctx, cm.addr(), cm.proxyURL, internalStateHook) if err != nil { return nil, err } - if tc, ok := pconn.conn.(*tls.Conn); ok { - cs := tc.ConnectionState() + return &http_persistConn{ + t: t, + cacheKey: cm.key(), + alt: rt, + }, nil + } + + pconn = &http_persistConn{ + t: t, + cacheKey: cm.key(), + reqch: make(chan http_requestAndChan, 1), + writech: make(chan http_writeRequest, 1), + closech: make(chan struct{}), + writeErrCh: make(chan error, 1), + writeLoopDone: make(chan struct{}), + isClientConn: isClientConn, + internalStateHook: internalStateHook, + } + trace := httptrace.ContextClientTrace(ctx) + wrapErr := func(err error) error { + if cm.proxyURL != nil { + // Return a typed error, per Issue 16997 + return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} + } + return err + } + + if rt, err := t.http2ExternalDial(ctx, cm); err != errors.ErrUnsupported { + if err != nil { + return nil, err + } + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: rt}, nil + } + + if cm.scheme() == "https" && t.hasCustomTLSDialer() { + var err error + pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) + if err != nil { + return nil, wrapErr(err) + } + type connectionStater interface { + ConnectionState() tls.ConnectionState + } + type handshaker interface { + HandshakeContext(context.Context) error + } + if cstater, ok := pconn.conn.(connectionStater); ok { + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + if handshaker, ok := cstater.(handshaker); ok { + // Handshake here, in case DialTLS didn't. TLSNextProto below + // depends on it for knowing the connection state. + if err := handshaker.HandshakeContext(ctx); err != nil { + go pconn.conn.Close() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } + return nil, err + } + } + cs := cstater.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } pconn.tlsState = &cs } } else { - conn, err := t.dial("tcp", cm.addr()) + conn, err := t.dial(ctx, "tcp", cm.addr()) if err != nil { - if cm.proxyURL != nil { - err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) - } - return nil, err + return nil, wrapErr(err) } pconn.conn = conn + if cm.scheme() == "https" { + var firstTLSHost string + if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { + return nil, wrapErr(err) + } + if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil { + return nil, wrapErr(err) + } + } } + // Proxy setup. switch { case cm.proxyURL == nil: - + // Do nothing. Not using a proxy. + case cm.proxyURL.Scheme == "socks5" || cm.proxyURL.Scheme == "socks5h": + conn := pconn.conn + d := http_socksNewDialer("tcp", conn.RemoteAddr().String()) + if u := cm.proxyURL.User; u != nil { + auth := &http_socksUsernamePassword{ + Username: u.Username(), + } + auth.Password, _ = u.Password() + d.AuthMethods = []http_socksAuthMethod{ + http_socksAuthMethodNotRequired, + http_socksAuthMethodUsernamePassword, + } + d.Authenticate = auth.Authenticate + } + if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { + conn.Close() + return nil, err + } case cm.targetScheme == "http": pconn.isProxy = true if pa := cm.proxyAuth(); pa != "" { - pconn.mutateHeaderFunc = func(h Header) { + pconn.mutateHeaderFunc = func(h http_Header) { h.Set("Proxy-Authorization", pa) } } case cm.targetScheme == "https": conn := pconn.conn - connectReq := &Request{ + var hdr http_Header + if t.GetProxyConnectHeader != nil { + var err error + hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr) + if err != nil { + conn.Close() + return nil, err + } + } else { + hdr = t.ProxyConnectHeader + } + if hdr == nil { + hdr = make(http_Header) + } + if pa := cm.proxyAuth(); pa != "" { + hdr = hdr.Clone() + hdr.Set("Proxy-Authorization", pa) + } + connectReq := &http_Request{ Method: "CONNECT", URL: &url.URL{Opaque: cm.targetAddr}, Host: cm.targetAddr, - Header: make(Header), + Header: hdr, } - if pa := cm.proxyAuth(); pa != "" { - connectReq.Header.Set("Proxy-Authorization", pa) - } - connectReq.Write(conn) - br := bufio.NewReader(conn) - resp, err := ReadResponse(br, connectReq) + // Set a (long) timeout here to make sure we don't block forever + // and leak a goroutine if the connection stops replying after + // the TCP connect. + connectCtx, cancel := http_testHookProxyConnectTimeout(ctx, 1*time.Minute) + defer cancel() + + didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails + var ( + resp *http_Response + err error // write or read error + ) + // Write the CONNECT request & read the response. + go func() { + defer close(didReadResponse) + err = connectReq.Write(conn) + if err != nil { + return + } + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(&io.LimitedReader{R: conn, N: t.maxHeaderResponseSize()}) + resp, err = http_ReadResponse(br, connectReq) + }() + select { + case <-connectCtx.Done(): + conn.Close() + <-didReadResponse + return nil, connectCtx.Err() + case <-didReadResponse: + // resp or err now set + } if err != nil { conn.Close() return nil, err } - if resp.StatusCode != 200 { - f := strings.SplitN(resp.Status, " ", 2) - conn.Close() - return nil, errors.New(f[1]) - } - } - if cm.targetScheme == "https" && !tlsDial { - - cfg := cloneTLSClientConfig(t.TLSClientConfig) - if cfg.ServerName == "" { - cfg.ServerName = cm.tlsHost() - } - plainConn := pconn.conn - tlsConn := tls.Client(plainConn, cfg) - errc := make(chan error, 2) - var timer *time.Timer // for canceling TLS handshake - if d := t.TLSHandshakeTimeout; d != 0 { - timer = time.AfterFunc(d, func() { - errc <- tlsHandshakeTimeoutError{} - }) - } - go func() { - err := tlsConn.Handshake() - if timer != nil { - timer.Stop() - } - errc <- err - }() - if err := <-errc; err != nil { - plainConn.Close() - return nil, err - } - if !cfg.InsecureSkipVerify { - if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { - plainConn.Close() + if t.OnProxyConnectResponse != nil { + err = t.OnProxyConnectResponse(ctx, cm.proxyURL, connectReq, resp) + if err != nil { + conn.Close() return nil, err } } - cs := tlsConn.ConnectionState() - pconn.tlsState = &cs - pconn.conn = tlsConn + + if resp.StatusCode != 200 { + _, text, ok := strings.Cut(resp.Status, " ") + conn.Close() + if !ok { + return nil, errors.New("unknown status code") + } + return nil, errors.New(text) + } } - pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) - pconn.bw = bufio.NewWriter(pconn.conn) + if cm.proxyURL != nil && cm.targetScheme == "https" { + if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil { + return nil, err + } + } + + // Possible unencrypted HTTP/2 with prior knowledge. + unencryptedHTTP2 := pconn.tlsState == nil && + t.Protocols != nil && + t.Protocols.UnencryptedHTTP2() && + !t.Protocols.HTTP1() + + http2 := unencryptedHTTP2 || + (pconn.tlsState != nil && pconn.tlsState.NegotiatedProtocol == "h2") + + if http2 && t.h2Transport != nil { + if isClientConn { + cc, err := t.http2NewClientConn(pconn.conn, internalStateHook) + if err == nil { + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: cc, isClientConn: true}, nil + } + if err != errors.ErrUnsupported { + return nil, err + } + } else { + rt, err := t.http2AddConn(cm.targetScheme, cm.targetAddr, pconn.conn) + if err == nil { + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: rt}, nil + } + if err != errors.ErrUnsupported { + return nil, err + } + } + } + + if isClientConn && (unencryptedHTTP2 || (pconn.tlsState != nil && pconn.tlsState.NegotiatedProtocol == "h2")) { + altProto, _ := t.altProto.Load().(map[string]http_RoundTripper) + h2, ok := altProto["https"].(http_newClientConner) + if !ok { + return nil, errors.New("http: HTTP/2 implementation does not support NewClientConn (update golang.org/x/net?)") + } + alt, err := h2.NewClientConn(pconn.conn, internalStateHook) + if err != nil { + pconn.conn.Close() + return nil, err + } + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt, isClientConn: true}, nil + } + + if unencryptedHTTP2 { + next, ok := t.TLSNextProto[http_nextProtoUnencryptedHTTP2] + if !ok { + return nil, errors.New("http: Transport does not support unencrypted HTTP/2") + } + alt := next(cm.targetAddr, http_unencryptedTLSConn(pconn.conn)) + if e, ok := alt.(http_erringRoundTripper); ok { + // pconn.conn was closed by next (http2configureTransports.upgradeFn). + return nil, e.RoundTripErr() + } + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil + } + + if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { + tlsConn, tlsConnOK := pconn.conn.(*tls.Conn) + if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; tlsConnOK && ok { + alt := next(cm.targetAddr, tlsConn) + if e, ok := alt.(http_erringRoundTripper); ok { + // pconn.conn was closed by next (http2configureTransports.upgradeFn). + return nil, e.RoundTripErr() + } + return &http_persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil + } + } + + pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) + pconn.bw = bufio.NewWriterSize(http_persistConnWriter{pconn}, t.writeBufferSize()) + go pconn.readLoop() go pconn.writeLoop() return pconn, nil } -// useProxy reports whether requests to addr should use a proxy, -// according to the NO_PROXY or no_proxy environment variable. -// addr is always a canonicalAddr with a host and port. -func useProxy(addr string) bool { - if len(addr) == 0 { - return true - } - host, _, err := net.SplitHostPort(addr) - if err != nil { - return false - } - if host == "localhost" { - return false - } - if ip := net.ParseIP(host); ip != nil { - if ip.IsLoopback() { - return false - } - } - - no_proxy := noProxyEnv.Get() - if no_proxy == "*" { - return false - } - - addr = strings.ToLower(strings.TrimSpace(addr)) - if hasPort(addr) { - addr = addr[:strings.LastIndex(addr, ":")] - } - - for _, p := range strings.Split(no_proxy, ",") { - p = strings.ToLower(strings.TrimSpace(p)) - if len(p) == 0 { - continue - } - if hasPort(p) { - p = p[:strings.LastIndex(p, ":")] - } - if addr == p { - return false - } - if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { - - return false - } - if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { - - return false - } - } - return true +// persistConnWriter is the io.Writer written to by pc.bw. +// It accumulates the number of bytes written to the underlying conn, +// so the retry logic can determine whether any bytes made it across +// the wire. +// This is exactly 1 pointer field wide so it can go into an interface +// without allocation. +type http_persistConnWriter struct { + pc *http_persistConn } +func (w http_persistConnWriter) Write(p []byte) (n int, err error) { + n, err = w.pc.conn.Write(p) + w.pc.nwrite += int64(n) + return +} + +// ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if +// the Conn implements io.ReaderFrom, it can take advantage of optimizations +// such as sendfile. +func (w http_persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { + n, err = io.Copy(w.pc.conn, r) + w.pc.nwrite += n + return +} + +var _ io.ReaderFrom = (*http_persistConnWriter)(nil) + // connectMethod is the map key (in its String form) for keeping persistent // TCP connections alive for subsequent HTTP requests. // // A connect method may be of the following types: // -// Cache key form Description -// ----------------- ------------------------- -// |http|foo.com http directly to server, no proxy -// |https|foo.com https directly to server, no proxy -// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com -// http://proxy.com|http http to proxy, http to anywhere after that -// -// Note: no support to https to the proxy yet. -// -type connectMethod struct { +// connectMethod.key().String() Description +// ------------------------------ ------------------------- +// |http|foo.com http directly to server, no proxy +// |https|foo.com https directly to server, no proxy +// |https,h1|foo.com https directly to server w/o HTTP/2, no proxy +// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com +// http://proxy.com|http http to proxy, http to anywhere after that +// socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com +// socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com +// https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com +// https://proxy.com|http https to proxy, http to anywhere after that +type http_connectMethod struct { + _ http_incomparable proxyURL *url.URL // nil for no proxy, else full proxy URL targetScheme string // "http" or "https" - targetAddr string // Not used if proxy + http targetScheme (4th example in table) + // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), + // then targetAddr is not included in the connect method key, because the socket can + // be reused for different targetAddr values. + targetAddr string + onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 } -func (cm *connectMethod) key() connectMethodKey { +func (cm *http_connectMethod) key() http_connectMethodKey { proxyStr := "" targetAddr := cm.targetAddr if cm.proxyURL != nil { proxyStr = cm.proxyURL.String() - if cm.targetScheme == "http" { + if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { targetAddr = "" } } - return connectMethodKey{ + return http_connectMethodKey{ proxy: proxyStr, scheme: cm.targetScheme, addr: targetAddr, + onlyH1: cm.onlyH1, } } -// addr returns the first hop "host:port" to which we need to TCP connect. -func (cm *connectMethod) addr() string { +// scheme returns the first hop scheme: http, https, or socks5 +func (cm *http_connectMethod) scheme() string { if cm.proxyURL != nil { - return canonicalAddr(cm.proxyURL) + return cm.proxyURL.Scheme + } + return cm.targetScheme +} + +// addr returns the first hop "host:port" to which we need to TCP connect. +func (cm *http_connectMethod) addr() string { + if cm.proxyURL != nil { + return http_canonicalAddr(cm.proxyURL) } return cm.targetAddr } // tlsHost returns the host name to match against the peer's // TLS certificate. -func (cm *connectMethod) tlsHost() string { +func (cm *http_connectMethod) tlsHost() string { h := cm.targetAddr - if hasPort(h) { - h = h[:strings.LastIndex(h, ":")] - } - return h + return http_removePort(h) } // connectMethodKey is the map key version of connectMethod, with a // stringified proxy URL (or the empty string) instead of a pointer to // a URL. -type connectMethodKey struct { +type http_connectMethodKey struct { proxy, scheme, addr string + onlyH1 bool } -func (k connectMethodKey) String() string { - - return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr) +func (k http_connectMethodKey) String() string { + // Only used by tests. + var h1 string + if k.onlyH1 { + h1 = ",h1" + } + return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) } // persistConn wraps a connection, usually a persistent one // (but may be used for non-keep-alive requests as well) -type persistConn struct { - t *Transport - cacheKey connectMethodKey - conn net.Conn - tlsState *tls.ConnectionState - br *bufio.Reader // from conn - sawEOF bool // whether we've seen EOF from conn; owned by readLoop - bw *bufio.Writer // to conn - reqch chan requestAndChan // written by roundTrip; read by readLoop - writech chan writeRequest // written by roundTrip; read by writeLoop - closech chan struct{} // closed when conn closed - isProxy bool +type http_persistConn struct { + // alt optionally specifies the TLS NextProto RoundTripper. + // This is used for HTTP/2 today and future protocols later. + // If it's non-nil, the rest of the fields are unused. + alt http_RoundTripper + + t *http_Transport + cacheKey http_connectMethodKey + conn net.Conn + tlsState *tls.ConnectionState + br *bufio.Reader // from conn + bw *bufio.Writer // to conn + nwrite int64 // bytes written + reqch chan http_requestAndChan // written by roundTrip; read by readLoop + writech chan http_writeRequest // written by roundTrip; read by writeLoop + closech chan struct{} // closed when conn closed + availch chan struct{} // ClientConn only: contains a value when conn is usable + isProxy bool + sawEOF bool // whether we've seen EOF from conn; owned by readLoop + isClientConn bool // whether this is a ClientConn (outside any pool) + readLimit int64 // bytes allowed to be read; owned by readLoop // writeErrCh passes the request write error (usually nil) // from the writeLoop goroutine to the readLoop which passes // it off to the res.Body reader, which then uses it to decide // whether or not a connection can be reused. Issue 7569. writeErrCh chan error - lk sync.Mutex // guards following fields + writeLoopDone chan struct{} // closed when write loop ends + + // Both guarded by Transport.idleMu: + idleAt time.Time // time it last become idle + idleTimer *time.Timer // holding an AfterFunc to close it + + mu sync.Mutex // guards following fields numExpectedResponses int - closed bool // whether conn has been closed - broken bool // an error has happened on this connection; marked broken so it's not reused. - canceled bool // whether this conn was broken due a CancelRequest + closed error // set non-nil when conn is closed, before closech is closed + canceledErr error // set non-nil if conn is canceled + reused bool // whether conn has had successful request/response and is being reused. + reserved bool // ClientConn only: concurrency slot reserved + inFlight bool // ClientConn only: request is in flight + internalStateHook func() // ClientConn state hook + // mutateHeaderFunc is an optional func to modify extra // headers on each outbound request before it's written. (the // original Request given to RoundTrip is not modified) - mutateHeaderFunc func(Header) + mutateHeaderFunc func(http_Header) +} + +func (pc *http_persistConn) maxHeaderResponseSize() int64 { + return pc.t.maxHeaderResponseSize() +} + +func (pc *http_persistConn) Read(p []byte) (n int, err error) { + if pc.readLimit <= 0 { + return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) + } + if int64(len(p)) > pc.readLimit { + p = p[:pc.readLimit] + } + n, err = pc.conn.Read(p) + if err == io.EOF { + pc.sawEOF = true + } + pc.readLimit -= int64(n) + return } // isBroken reports whether this connection is in a known broken state. -func (pc *persistConn) isBroken() bool { - pc.lk.Lock() - b := pc.broken - pc.lk.Unlock() +func (pc *http_persistConn) isBroken() bool { + pc.mu.Lock() + b := pc.closed != nil + pc.mu.Unlock() return b } -// isCanceled reports whether this connection was closed due to CancelRequest. -func (pc *persistConn) isCanceled() bool { - pc.lk.Lock() - defer pc.lk.Unlock() - return pc.canceled +// canceled returns non-nil if the connection was closed due to +// CancelRequest or due to context cancellation. +func (pc *http_persistConn) canceled() error { + pc.mu.Lock() + defer pc.mu.Unlock() + return pc.canceledErr } -func (pc *persistConn) cancelRequest() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.canceled = true - pc.closeLocked() +// isReused reports whether this connection has been used before. +func (pc *http_persistConn) isReused() bool { + pc.mu.Lock() + r := pc.reused + pc.mu.Unlock() + return r } -func (pc *persistConn) readLoop() { +func (pc *http_persistConn) cancelRequest(err error) { + pc.mu.Lock() + defer pc.mu.Unlock() + pc.canceledErr = err + pc.closeLocked(http_errRequestCanceled) +} +// closeConnIfStillIdle closes the connection if it's still sitting idle. +// This is what's called by the persistConn's idleTimer, and is run in its +// own goroutine. +func (pc *http_persistConn) closeConnIfStillIdle() { + t := pc.t + t.idleMu.Lock() + defer t.idleMu.Unlock() + if _, ok := t.idleLRU.m[pc]; !ok { + // Not idle. + return + } + t.removeIdleConnLocked(pc) + pc.close(http_errIdleConnTimeout) +} + +// mapRoundTripError returns the appropriate error value for +// persistConn.roundTrip. +// +// The provided err is the first error that (*persistConn).roundTrip +// happened to receive from its select statement. +// +// The startBytesWritten value should be the value of pc.nwrite before the roundTrip +// started writing the request. +func (pc *http_persistConn) mapRoundTripError(req *http_transportRequest, startBytesWritten int64, err error) error { + if err == nil { + return nil + } + + // Wait for the writeLoop goroutine to terminate to avoid data + // races on callers who mutate the request on failure. + // + // When resc in pc.roundTrip and hence rc.ch receives a responseAndError + // with a non-nil error it implies that the persistConn is either closed + // or closing. Waiting on pc.writeLoopDone is hence safe as all callers + // close closech which in turn ensures writeLoop returns. + <-pc.writeLoopDone + + // If the request was canceled, that's better than network + // failures that were likely the result of tearing down the + // connection. + if cerr := pc.canceled(); cerr != nil { + return cerr + } + + // See if an error was set explicitly. + req.mu.Lock() + reqErr := req.err + req.mu.Unlock() + if reqErr != nil { + return reqErr + } + + if err == http_errServerClosedIdle { + // Don't decorate + return err + } + + if _, ok := err.(http_transportReadFromServerError); ok { + if pc.nwrite == startBytesWritten { + return http_nothingWrittenError{err} + } + // Don't decorate + return err + } + if pc.isBroken() { + if pc.nwrite == startBytesWritten { + return http_nothingWrittenError{err} + } + return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %w", err) + } + return err +} + +// errCallerOwnsConn is an internal sentinel error used when we hand +// off a writable response.Body to the caller. We use this to prevent +// closing a net.Conn that is now owned by the caller. +var http_errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") + +// maxPostCloseReadBytes is the max number of bytes that a client is willing to +// read when draining the response body of any unread bytes after it has been +// closed. This number is chosen for consistency with maxPostHandlerReadBytes. +const http_maxPostCloseReadBytes = 256 << 10 + +// maxPostCloseReadTime defines the maximum amount of time that a client is +// willing to spend on draining a response body of any unread bytes after it +// has been closed. +const http_maxPostCloseReadTime = 50 * time.Millisecond + +func http_maybeDrainBody(body io.Reader) bool { + drainedCh := make(chan bool, 1) + go func() { + if _, err := io.CopyN(io.Discard, body, http_maxPostCloseReadBytes+1); err == io.EOF { + drainedCh <- true + } else { + drainedCh <- false + } + }() + select { + case drained := <-drainedCh: + return drained + case <-time.After(http_maxPostCloseReadTime): + return false + } +} + +func (pc *http_persistConn) readLoop() { + closeErr := http_errReadLoopExiting // default value, if not changed below + defer func() { + pc.close(closeErr) + pc.t.removeIdleConn(pc) + if pc.internalStateHook != nil { + pc.internalStateHook() + } + }() + + tryPutIdleConn := func(treq *http_transportRequest) bool { + trace := treq.trace + if err := pc.t.tryPutIdleConn(pc); err != nil { + closeErr = err + if trace != nil && trace.PutIdleConn != nil && err != http_errKeepAlivesDisabled { + trace.PutIdleConn(err) + } + return false + } + if trace != nil && trace.PutIdleConn != nil { + trace.PutIdleConn(nil) + } + return true + } + + // eofc is used to block caller goroutines reading from Response.Body + // at EOF until this goroutines has (potentially) added the connection + // back to the idle pool. eofc := make(chan struct{}) - defer close(eofc) + defer close(eofc) // unblock reader on errors - testHookMu.Lock() - testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead - testHookMu.Unlock() + // Read this once, before loop starts. (to avoid races in tests) + http_testHookMu.Lock() + testHookReadLoopBeforeNextRead := http_testHookReadLoopBeforeNextRead + http_testHookMu.Unlock() alive := true for alive { - pb, err := pc.br.Peek(1) + pc.readLimit = pc.maxHeaderResponseSize() + _, err := pc.br.Peek(1) - pc.lk.Lock() + pc.mu.Lock() if pc.numExpectedResponses == 0 { - if !pc.closed { - pc.closeLocked() - if len(pb) > 0 { - log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", - string(pb), err) - } - } - pc.lk.Unlock() + pc.readLoopPeekFailLocked(err) + pc.mu.Unlock() return } - pc.lk.Unlock() + pc.mu.Unlock() rc := <-pc.reqch + trace := rc.treq.trace - var resp *Response + var resp *http_Response if err == nil { - resp, err = ReadResponse(pc.br, rc.req) - if err == nil && resp.StatusCode == 100 { - - resp, err = ReadResponse(pc.br, rc.req) - } + resp, err = pc.readResponse(rc, trace) + } else { + err = http_transportReadFromServerError{err} + closeErr = err } - if resp != nil { - resp.TLS = pc.tlsState - } - - hasBody := resp != nil && rc.req.Method != "HEAD" && resp.ContentLength != 0 - if err != nil { - pc.close() - } else { - if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" { - resp.Header.Del("Content-Encoding") - resp.Header.Del("Content-Length") - resp.ContentLength = -1 - resp.Body = &gzipReader{body: resp.Body} + if pc.readLimit <= 0 { + err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) } - resp.Body = &bodyEOFSignal{body: resp.Body} + + select { + case rc.ch <- http_responseAndError{err: err}: + case <-rc.callerGone: + return + } + return } + pc.readLimit = http_maxInt64 // effectively no limit for response bodies - if err != nil || resp.Close || rc.req.Close || resp.StatusCode <= 199 { + pc.mu.Lock() + pc.numExpectedResponses-- + pc.mu.Unlock() + bodyWritable := resp.bodyIsWritable() + hasBody := rc.treq.http_Request.Method != "HEAD" && resp.ContentLength != 0 + + if resp.Close || rc.treq.http_Request.Close || resp.StatusCode <= 199 || bodyWritable { + // Don't do keep-alive on error if either party requested a close + // or we get an unexpected informational (1xx) response. + // StatusCode 100 is already handled above. alive = false } - var waitForBodyRead chan bool // channel is nil when there's no body - if hasBody { - waitForBodyRead = make(chan bool, 2) - resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { - waitForBodyRead <- false - return nil - } - resp.Body.(*bodyEOFSignal).fn = func(err error) error { - isEOF := err == io.EOF - waitForBodyRead <- isEOF - if isEOF { - <-eofc - } else if err != nil && pc.isCanceled() { - return errRequestCanceled - } - return err - } - } else { - - pc.t.setReqCanceler(rc.req, nil) - } - - pc.lk.Lock() - pc.numExpectedResponses-- - pc.lk.Unlock() - - rc.ch <- responseAndError{resp, err} - - if hasBody { - - select { - case <-rc.req.Cancel: - alive = false - pc.t.CancelRequest(rc.req) - case bodyEOF := <-waitForBodyRead: - pc.t.setReqCanceler(rc.req, nil) - alive = alive && - bodyEOF && - !pc.sawEOF && - pc.wroteRequest() && - pc.t.putIdleConn(pc) - if bodyEOF { - eofc <- struct{}{} - } - case <-pc.closech: - alive = false - } - } else { + if !hasBody || bodyWritable { + // Put the idle conn back into the pool before we send the response + // so if they process it quickly and make another request, they'll + // get this same conn. But we use the unbuffered channel 'rc' + // to guarantee that persistConn.roundTrip got out of its select + // potentially waiting for this persistConn to close. alive = alive && !pc.sawEOF && pc.wroteRequest() && - pc.t.putIdleConn(pc) + tryPutIdleConn(rc.treq) + + if bodyWritable { + closeErr = http_errCallerOwnsConn + } + + select { + case rc.ch <- http_responseAndError{res: resp}: + case <-rc.callerGone: + return + } + + rc.treq.cancel(http_errRequestDone) + + // Now that they've read from the unbuffered channel, they're safely + // out of the select that also waits on this goroutine to die, so + // we're allowed to exit now if needed (if alive is false) + testHookReadLoopBeforeNextRead() + continue } - if hook := testHookReadLoopBeforeNextRead; hook != nil { - hook() - } - } - pc.close() -} + waitForBodyRead := make(chan bool, 2) + body := &http_bodyEOFSignal{ + body: resp.Body, + earlyCloseFn: func() error { + waitForBodyRead <- false + <-eofc // will be closed by deferred call at the end of the function + return nil -func (pc *persistConn) writeLoop() { - for { + }, + fn: func(err error) error { + isEOF := err == io.EOF + waitForBodyRead <- isEOF + if isEOF { + <-eofc // see comment above eofc declaration + } else if err != nil { + if cerr := pc.canceled(); cerr != nil { + return cerr + } + } + return err + }, + } + + resp.Body = body + if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { + resp.Body = &http_gzipReader{body: body} + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + resp.Uncompressed = true + } + select { - case wr := <-pc.writech: - if pc.isBroken() { - wr.ch <- errors.New("http: can't write HTTP request on broken connection") - continue - } - err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra) - if err == nil { - err = pc.bw.Flush() - } - if err != nil { - pc.markBroken() - wr.req.Request.closeBody() - } - pc.writeErrCh <- err - wr.ch <- err - case <-pc.closech: + case rc.ch <- http_responseAndError{res: resp}: + case <-rc.callerGone: return } + + // Before looping back to the top of this function and peeking on + // the bufio.Reader, wait for the caller goroutine to finish + // reading the response body. (or for cancellation or death) + select { + case bodyEOF := <-waitForBodyRead: + tryDrain := !bodyEOF && resp.ContentLength <= http_maxPostCloseReadBytes + if tryDrain { + eofc <- struct{}{} + bodyEOF = http_maybeDrainBody(body.body) + } + alive = alive && + bodyEOF && + !pc.sawEOF && + pc.wroteRequest() && + tryPutIdleConn(rc.treq) + if !tryDrain && bodyEOF { + eofc <- struct{}{} + } + case <-rc.treq.ctx.Done(): + alive = false + pc.cancelRequest(context.Cause(rc.treq.ctx)) + case <-pc.closech: + alive = false + } + + rc.treq.cancel(http_errRequestDone) + testHookReadLoopBeforeNextRead() } } -// wroteRequest is a check before recycling a connection that the previous write -// (from writeLoop above) happened and was successful. -func (pc *persistConn) wroteRequest() bool { - select { - case err := <-pc.writeErrCh: +func (pc *http_persistConn) readLoopPeekFailLocked(peekErr error) { + if pc.closed != nil { + return + } + if n := pc.br.Buffered(); n > 0 { + buf, _ := pc.br.Peek(n) + if http_is408Message(buf) { + pc.closeLocked(http_errServerClosedIdle) + return + } else { + log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) + } + } + if peekErr == io.EOF { + // common case. + pc.closeLocked(http_errServerClosedIdle) + } else { + pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %w", peekErr)) + } +} - return err == nil - default: +// is408Message reports whether buf has the prefix of an +// HTTP 408 Request Timeout response. +// See golang.org/issue/32310. +func http_is408Message(buf []byte) bool { + if len(buf) < len("HTTP/1.x 408") { + return false + } + if string(buf[:7]) != "HTTP/1." { + return false + } + return string(buf[8:12]) == " 408" +} + +// readResponse reads an HTTP response (or two, in the case of "Expect: +// 100-continue") from the server. It returns the final non-100 one. +// trace is optional. +func (pc *http_persistConn) readResponse(rc http_requestAndChan, trace *httptrace.ClientTrace) (resp *http_Response, err error) { + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } + + continueCh := rc.continueCh + for { + resp, err = http_ReadResponse(pc.br, rc.treq.http_Request) + if err != nil { + return + } + resCode := resp.StatusCode + if continueCh != nil && resCode == http_StatusContinue { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } + continueCh <- struct{}{} + continueCh = nil + } + is1xx := 100 <= resCode && resCode <= 199 + // treat 101 as a terminal status, see issue 26161 + is1xxNonTerminal := is1xx && resCode != http_StatusSwitchingProtocols + if is1xxNonTerminal { + if trace != nil && trace.Got1xxResponse != nil { + if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { + return nil, err + } + // If the 1xx response was delivered to the user, + // then they're responsible for limiting the number of + // responses. Reset the header limit. + // + // If the user didn't examine the 1xx response, then we + // limit the size of all headers (including both 1xx + // and the final response) to maxHeaderResponseSize. + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit + } + continue + } + break + } + if resp.isProtocolSwitch() { + resp.Body = http_newReadWriteCloserBody(pc.br, pc.conn) + } + if continueCh != nil { + // We send an "Expect: 100-continue" header, but the server + // responded with a terminal status and no 100 Continue. + // + // If we're going to keep using the connection, we need to send the request body. + // Tell writeLoop to skip sending the body if we're going to close the connection, + // or to send it otherwise. + // + // The case where we receive a 101 Switching Protocols response is a bit + // ambiguous, since we don't know what protocol we're switching to. + // Conceivably, it's one that doesn't need us to send the body. + // Given that we'll send the body if ExpectContinueTimeout expires, + // be consistent and always send it if we aren't closing the connection. + if resp.Close || rc.treq.http_Request.Close { + close(continueCh) // don't send the body; the connection will close + } else { + continueCh <- struct{}{} // send the body + } + } + + resp.TLS = pc.tlsState + return +} + +// waitForContinue returns the function to block until +// any response, timeout or connection close. After any of them, +// the function returns a bool which indicates if the body should be sent. +func (pc *http_persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { + if continueCh == nil { + return nil + } + return func() bool { + timer := time.NewTimer(pc.t.ExpectContinueTimeout) + defer timer.Stop() select { - case err := <-pc.writeErrCh: - return err == nil - case <-time.After(50 * time.Millisecond): + case _, ok := <-continueCh: + return ok + case <-timer.C: + return true + case <-pc.closech: return false } } } -type responseAndError struct { - res *Response +func http_newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { + body := &http_readWriteCloserBody{ReadWriteCloser: rwc} + if br.Buffered() != 0 { + body.br = br + } + return body +} + +// readWriteCloserBody is the Response.Body type used when we want to +// give users write access to the Body through the underlying +// connection (TCP, unless using custom dialers). This is then +// the concrete type for a Response.Body on the 101 Switching +// Protocols response, as used by WebSockets, h2c, etc. +type http_readWriteCloserBody struct { + _ http_incomparable + br *bufio.Reader // used until empty + io.ReadWriteCloser +} + +func (b *http_readWriteCloserBody) Read(p []byte) (n int, err error) { + if b.br != nil { + if n := b.br.Buffered(); len(p) > n { + p = p[:n] + } + n, err = b.br.Read(p) + if b.br.Buffered() == 0 { + b.br = nil + } + return n, err + } + return b.ReadWriteCloser.Read(p) +} + +func (b *http_readWriteCloserBody) CloseWrite() error { + if cw, ok := b.ReadWriteCloser.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return fmt.Errorf("CloseWrite: %w", http_ErrNotSupported) +} + +// nothingWrittenError wraps a write errors which ended up writing zero bytes. +type http_nothingWrittenError struct { + error +} + +func (nwe http_nothingWrittenError) Unwrap() error { + return nwe.error +} + +func (pc *http_persistConn) writeLoop() { + defer close(pc.writeLoopDone) + for { + select { + case wr := <-pc.writech: + startBytesWritten := pc.nwrite + err := wr.req.http_Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) + if bre, ok := err.(http_requestBodyReadError); ok { + err = bre.error + // Errors reading from the user's + // Request.Body are high priority. + // Set it here before sending on the + // channels below or calling + // pc.close() which tears down + // connections and causes other + // errors. + wr.req.setError(err) + } + if err == nil { + err = pc.bw.Flush() + } + if err != nil { + if pc.nwrite == startBytesWritten { + err = http_nothingWrittenError{err} + } + } + pc.writeErrCh <- err // to the body reader, which might recycle us + wr.ch <- err // to the roundTrip function + if err != nil { + pc.close(err) + return + } + case <-pc.closech: + return + } + } +} + +// maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip +// will wait to see the Request's Body.Write result after getting a +// response from the server. See comments in (*persistConn).wroteRequest. +// +// In tests, we set this to a large value to avoid flakiness from inconsistent +// recycling of connections. +var http_maxWriteWaitBeforeConnReuse = 50 * time.Millisecond + +// wroteRequest is a check before recycling a connection that the previous write +// (from writeLoop above) happened and was successful. +func (pc *http_persistConn) wroteRequest() bool { + select { + case err := <-pc.writeErrCh: + // Common case: the write happened well before the response, so + // avoid creating a timer. + return err == nil + default: + // Rare case: the request was written in writeLoop above but + // before it could send to pc.writeErrCh, the reader read it + // all, processed it, and called us here. In this case, give the + // write goroutine a bit of time to finish its send. + // + // Less rare case: We also get here in the legitimate case of + // Issue 7569, where the writer is still writing (or stalled), + // but the server has already replied. In this case, we don't + // want to wait too long, and we want to return false so this + // connection isn't re-used. + t := time.NewTimer(http_maxWriteWaitBeforeConnReuse) + defer t.Stop() + select { + case err := <-pc.writeErrCh: + return err == nil + case <-t.C: + return false + } + } +} + +// responseAndError is how the goroutine reading from an HTTP/1 server +// communicates with the goroutine doing the RoundTrip. +type http_responseAndError struct { + _ http_incomparable + res *http_Response // else use this response (see res method) err error } -type requestAndChan struct { - req *Request - ch chan responseAndError +type http_requestAndChan struct { + _ http_incomparable + treq *http_transportRequest + ch chan http_responseAndError // unbuffered; always send in select on callerGone - // did the Transport (as opposed to the client code) add an - // Accept-Encoding gzip header? only if it we set it do - // we transparently decode the gzip. + // whether the Transport (as opposed to the user client code) + // added the Accept-Encoding gzip header. If the Transport + // set it, only then do we transparently decode the gzip. addedGzip bool + + // Optional blocking chan for Expect: 100-continue (for send). + // If the request has an "Expect: 100-continue" header and + // the server responds 100 Continue, readLoop send a value + // to writeLoop via this chan. + continueCh chan<- struct{} + + callerGone <-chan struct{} // closed when roundTrip caller has returned } -// A writeRequest is sent by the readLoop's goroutine to the +// A writeRequest is sent by the caller's goroutine to the // writeLoop's goroutine to write a request while the read loop // concurrently waits on both the write response and the server's // reply. -type writeRequest struct { - req *transportRequest +type http_writeRequest struct { + req *http_transportRequest ch chan<- error + + // Optional blocking chan for Expect: 100-continue (for receive). + // If not nil, writeLoop blocks sending request body until + // it receives from this chan. + continueCh <-chan struct{} } -type httpError struct { - err string - timeout bool +// httpTimeoutError represents a timeout. +// It implements net.Error and wraps context.DeadlineExceeded. +type http_timeoutError struct { + err string } -func (e *httpError) Error() string { return e.err } +func (e *http_timeoutError) Error() string { return e.err } -func (e *httpError) Timeout() bool { return e.timeout } +func (e *http_timeoutError) Timeout() bool { return true } -func (e *httpError) Temporary() bool { return true } +func (e *http_timeoutError) Temporary() bool { return true } -var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} +func (e *http_timeoutError) Is(err error) bool { return err == context.DeadlineExceeded } -var errClosed error = &httpError{err: "net/http: transport closed before response was received"} +var http_errTimeout error = &http_timeoutError{"net/http: timeout awaiting response headers"} -var errRequestCanceled = errors.New("net/http: request canceled") +// errRequestCanceled is set to be identical to the one from h2 to facilitate +// testing. +var http_errRequestCanceled = internal.ErrRequestCanceled -// nil except for tests +var http_errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? + +// errRequestDone is used to cancel the round trip Context after a request is successfully done. +// It should not be seen by the user. +var http_errRequestDone = errors.New("net/http: request completed") + +func http_nop() {} + +// testHooks. Always non-nil. var ( - testHookPersistConnClosedGotRes func() - testHookEnterRoundTrip func() - testHookMu sync.Locker = fakeLocker{} // guards following - testHookReadLoopBeforeNextRead func() + http_testHookEnterRoundTrip = http_nop + http_testHookWaitResLoop = http_nop + http_testHookRoundTripRetried = http_nop + http_testHookPrePendingDial = http_nop + http_testHookPostPendingDial = http_nop + + http_testHookMu sync.Locker = http_fakeLocker{} // guards following + http_testHookReadLoopBeforeNextRead = http_nop ) -func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { - if hook := testHookEnterRoundTrip; hook != nil { - hook() +func (pc *http_persistConn) waitForAvailability(ctx context.Context) error { + select { + case <-pc.availch: + return nil + case <-pc.closech: + return pc.closed + case <-ctx.Done(): + return ctx.Err() } - if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { - pc.t.putIdleConn(pc) - return nil, errRequestCanceled +} + +func (pc *http_persistConn) roundTrip(req *http_transportRequest) (resp *http_Response, err error) { + http_testHookEnterRoundTrip() + + pc.mu.Lock() + if pc.isClientConn { + if !pc.reserved { + pc.mu.Unlock() + if err := pc.waitForAvailability(req.ctx); err != nil { + return nil, err + } + pc.mu.Lock() + } + pc.reserved = false + pc.inFlight = true } - pc.lk.Lock() pc.numExpectedResponses++ headerFn := pc.mutateHeaderFunc - pc.lk.Unlock() + pc.mu.Unlock() if headerFn != nil { headerFn(req.extraHeaders()) } + // Ask for a compressed version if the caller didn't set their + // own value for Accept-Encoding. We only attempt to + // uncompress the gzip stream if we were the layer that + // requested it. requestedGzip := false if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" && req.Header.Get("Range") == "" && req.Method != "HEAD" { - + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: https://zlib.net/zlib_faq.html#faq39 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // https://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 requestedGzip = true req.extraHeaders().Set("Accept-Encoding", "gzip") } - if pc.t.DisableKeepAlives { + var continueCh chan struct{} + if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { + continueCh = make(chan struct{}, 1) + } + + if pc.t.DisableKeepAlives && + !req.wantsClose() && + !http_isProtocolSwitchHeader(req.Header) { req.extraHeaders().Set("Connection", "close") } + gone := make(chan struct{}) + defer close(gone) + + const debugRoundTrip = false + + // Write the request concurrently with waiting for a response, + // in case the server decides to reply before reading our full + // request body. + startBytesWritten := pc.nwrite writeErrCh := make(chan error, 1) - pc.writech <- writeRequest{req, writeErrCh} + pc.writech <- http_writeRequest{req, writeErrCh, continueCh} - resc := make(chan responseAndError, 1) - pc.reqch <- requestAndChan{req.Request, resc, requestedGzip} + resc := make(chan http_responseAndError) + pc.reqch <- http_requestAndChan{ + treq: req, + ch: resc, + addedGzip: requestedGzip, + continueCh: continueCh, + callerGone: gone, + } - var re responseAndError + handleResponse := func(re http_responseAndError) (*http_Response, error) { + if (re.res == nil) == (re.err == nil) { + panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) + } + if debugRoundTrip { + req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) + } + if re.err != nil { + return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) + } + return re.res, nil + } + var respHeaderTimer <-chan time.Time - cancelChan := req.Request.Cancel -WaitResponse: + ctxDoneChan := req.ctx.Done() + pcClosed := pc.closech for { + http_testHookWaitResLoop() select { case err := <-writeErrCh: - if isNetWriteError(err) { - - select { - case re = <-resc: - pc.close() - break WaitResponse - case <-time.After(50 * time.Millisecond): - - } + if debugRoundTrip { + req.logf("writeErrCh recv: %T/%#v", err, err) } if err != nil { - re = responseAndError{nil, err} - pc.close() - break WaitResponse + pc.close(fmt.Errorf("write error: %w", err)) + return nil, pc.mapRoundTripError(req, startBytesWritten, err) } if d := pc.t.ResponseHeaderTimeout; d > 0 { + if debugRoundTrip { + req.logf("starting timer for %v", d) + } timer := time.NewTimer(d) - defer timer.Stop() + defer timer.Stop() // prevent leaks respHeaderTimer = timer.C } - case <-pc.closech: - + case <-pcClosed: select { - case re = <-resc: - if fn := testHookPersistConnClosedGotRes; fn != nil { - fn() - } + case re := <-resc: + // The pconn closing raced with the response to the request, + // probably after the server wrote a response and immediately + // closed the connection. Use the response. + return handleResponse(re) default: - re = responseAndError{err: errClosed} - if pc.isCanceled() { - re = responseAndError{err: errRequestCanceled} - } } - break WaitResponse + if debugRoundTrip { + req.logf("closech recv: %T %#v", pc.closed, pc.closed) + } + return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) case <-respHeaderTimer: - pc.close() - re = responseAndError{err: errTimeout} - break WaitResponse - case re = <-resc: - break WaitResponse - case <-cancelChan: - pc.t.CancelRequest(req.Request) - cancelChan = nil + if debugRoundTrip { + req.logf("timeout waiting for response headers.") + } + pc.close(http_errTimeout) + return nil, http_errTimeout + case re := <-resc: + return handleResponse(re) + case <-ctxDoneChan: + select { + case re := <-resc: + // readLoop is responsible for canceling req.ctx after + // it reads the response body. Check for a response racing + // the context close, and use the response if available. + return handleResponse(re) + default: + } + pc.cancelRequest(context.Cause(req.ctx)) } } +} - if re.err != nil { - pc.t.setReqCanceler(req.Request, nil) +// tLogKey is a context WithValue key for test debugging contexts containing +// a t.Logf func. See export_test.go's Request.WithT method. +type http_tLogKey struct{} + +func (tr *http_transportRequest) logf(format string, args ...any) { + if logf, ok := tr.http_Request.Context().Value(http_tLogKey{}).(func(string, ...any)); ok { + logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) } - return re.res, re.err } -// markBroken marks a connection as broken (so it's not reused). -// It differs from close in that it doesn't close the underlying -// connection for use when it's still being read. -func (pc *persistConn) markBroken() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.broken = true +// markReused marks this connection as having been successfully used for a +// request and response. +func (pc *http_persistConn) markReused() { + pc.mu.Lock() + pc.reused = true + pc.mu.Unlock() } -func (pc *persistConn) close() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.closeLocked() +// close closes the underlying TCP connection and closes +// the pc.closech channel. +// +// The provided err is only for testing and debugging; in normal +// circumstances it should never be seen by users. +func (pc *http_persistConn) close(err error) { + pc.mu.Lock() + defer pc.mu.Unlock() + pc.closeLocked(err) } -func (pc *persistConn) closeLocked() { - pc.broken = true - if !pc.closed { - pc.conn.Close() - pc.closed = true - close(pc.closech) +func (pc *http_persistConn) closeLocked(err error) { + if err == nil { + panic("nil error") + } + if pc.closed == nil { + pc.closed = err + pc.t.decConnsPerHost(pc.cacheKey) + // Close HTTP/1 (pc.alt == nil) connection. + // HTTP/2 closes its connection itself. + // Close HTTP/3 connection if it implements io.Closer. + if pc.alt == nil { + if err != http_errCallerOwnsConn { + pc.conn.Close() + } + close(pc.closech) + } else { + if cc, ok := pc.alt.(io.Closer); ok { + cc.Close() + } + } } pc.mutateHeaderFunc = nil } -var portMap = map[string]string{ - "http": "80", - "https": "443", +func http_schemePort(scheme string) string { + switch scheme { + case "http": + return "80" + case "https": + return "443" + case "socks5", "socks5h": + return "1080" + default: + return "" + } } -// canonicalAddr returns url.Host but always with a ":port" suffix -func canonicalAddr(url *url.URL) string { - addr := url.Host - if !hasPort(addr) { - return addr + ":" + portMap[url.Scheme] +func http_idnaASCIIFromURL(url *url.URL) string { + addr := url.Hostname() + if v, err := http_idnaASCII(addr); err == nil { + addr = v } return addr } -// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most +// canonicalAddr returns url.Host but always with a ":port" suffix. +func http_canonicalAddr(url *url.URL) string { + port := url.Port() + if port == "" { + port = http_schemePort(url.Scheme) + } + return net.JoinHostPort(http_idnaASCIIFromURL(url), port) +} + +// bodyEOFSignal is used by the HTTP/1 transport when reading response +// bodies to make sure we see the end of a response body before +// proceeding and reading on the connection again. +// +// It wraps a ReadCloser but runs fn (if non-nil) at most // once, right before its final (error-producing) Read or Close call // returns. fn should return the new error to return from Read or Close. // // If earlyCloseFn is non-nil and Close is called before io.EOF is // seen, earlyCloseFn is called instead of fn, and its return value is // the return value from Close. -type bodyEOFSignal struct { +type http_bodyEOFSignal struct { body io.ReadCloser mu sync.Mutex // guards following 4 fields closed bool // whether Close has been called @@ -7517,12 +17351,16 @@ earlyCloseFn func() error // optional alt Close func used if io.EOF not seen } -func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { +var http_errReadOnClosedResBody = errors.New("http: read on closed response body") + +var http_errConcurrentReadOnResBody = errors.New("http: concurrent read on response body") + +func (es *http_bodyEOFSignal) Read(p []byte) (n int, err error) { es.mu.Lock() closed, rerr := es.closed, es.rerr es.mu.Unlock() if closed { - return 0, errors.New("http: read on closed response body") + return 0, http_errReadOnClosedResBody } if rerr != nil { return 0, rerr @@ -7540,7 +17378,7 @@ return } -func (es *bodyEOFSignal) Close() error { +func (es *http_bodyEOFSignal) Close() error { es.mu.Lock() defer es.mu.Unlock() if es.closed { @@ -7555,7 +17393,7 @@ } // caller must hold es.mu. -func (es *bodyEOFSignal) condfn(err error) error { +func (es *http_bodyEOFSignal) condfn(err error) error { if es.fn == nil { return err } @@ -7565,136 +17403,194 @@ } // gzipReader wraps a response body so it can lazily -// call gzip.NewReader on the first call to Read -type gzipReader struct { - body io.ReadCloser // underlying Response.Body - zr io.Reader // lazily-initialized gzip reader +// get gzip.Reader from the pool on the first call to Read. +// After Close is called it puts gzip.Reader to the pool immediately +// if there is no Read in progress or later when Read completes. +type http_gzipReader struct { + _ http_incomparable + body *http_bodyEOFSignal // underlying HTTP/1 response body framing + mu sync.Mutex // guards zr and zerr + zr *gzip.Reader // stores gzip reader from the pool between reads + zerr error // sticky gzip reader init error or sentinel value to detect concurrent read and read after close } -func (gz *gzipReader) Read(p []byte) (n int, err error) { +type http_eofReader struct{} + +func (http_eofReader) Read([]byte) (int, error) { return 0, io.EOF } + +func (http_eofReader) ReadByte() (byte, error) { return 0, io.EOF } + +var http_gzipPool = sync.Pool{New: func() any { return new(gzip.Reader) }} + +// gzipPoolGet gets a gzip.Reader from the pool and resets it to read from r. +func http_gzipPoolGet(r io.Reader) (*gzip.Reader, error) { + zr := http_gzipPool.Get().(*gzip.Reader) + if err := zr.Reset(r); err != nil { + http_gzipPoolPut(zr) + return nil, err + } + return zr, nil +} + +// gzipPoolPut puts a gzip.Reader back into the pool. +func http_gzipPoolPut(zr *gzip.Reader) { + // Reset will allocate bufio.Reader if we pass it anything + // other than a flate.Reader, so ensure that it's getting one. + var r flate.Reader = http_eofReader{} + zr.Reset(r) + http_gzipPool.Put(zr) +} + +// acquire returns a gzip.Reader for reading response body. +// The reader must be released after use. +func (gz *http_gzipReader) acquire() (*gzip.Reader, error) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr != nil { + return nil, gz.zerr + } if gz.zr == nil { - gz.zr, err = gzip.NewReader(gz.body) - if err != nil { - return 0, err + // gzipPoolGet might block indefinitely since it reads the gzip header. + // Therefore, drop mu temporarily when using gzipPoolGet. + // We set zerr to errConcurrentReadOnResBody to prevent concurrent read + // even when mu is temporarily dropped. + gz.zerr = http_errConcurrentReadOnResBody + gz.mu.Unlock() + zr, err := http_gzipPoolGet(gz.body) + gz.mu.Lock() + // Guard against Close being called while gzipPoolGet is running. + if gz.zerr != http_errConcurrentReadOnResBody { + if zr != nil { + http_gzipPoolPut(zr) + } + return nil, gz.zerr + } + gz.zr, gz.zerr = zr, err + if gz.zerr != nil { + return nil, gz.zerr } } - return gz.zr.Read(p) + ret := gz.zr + gz.zr, gz.zerr = nil, http_errConcurrentReadOnResBody + return ret, nil } -func (gz *gzipReader) Close() error { +// release returns the gzip.Reader to the pool if Close was called during Read. +func (gz *http_gzipReader) release(zr *gzip.Reader) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == http_errConcurrentReadOnResBody { + gz.zr, gz.zerr = zr, nil + } else { // errReadOnClosedResBody + http_gzipPoolPut(zr) + } +} + +// close returns the gzip.Reader to the pool immediately or +// signals release to do so after Read completes. +func (gz *http_gzipReader) close() { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == nil && gz.zr != nil { + http_gzipPoolPut(gz.zr) + gz.zr = nil + } + gz.zerr = http_errReadOnClosedResBody +} + +func (gz *http_gzipReader) Read(p []byte) (n int, err error) { + zr, err := gz.acquire() + if err != nil { + return 0, err + } + defer gz.release(zr) + + return zr.Read(p) +} + +func (gz *http_gzipReader) Close() error { + gz.close() + return gz.body.Close() } -type readerAndCloser struct { - io.Reader - io.Closer -} +type http_tlsHandshakeTimeoutError struct{} -type tlsHandshakeTimeoutError struct{} +func (http_tlsHandshakeTimeoutError) Timeout() bool { return true } -func (tlsHandshakeTimeoutError) Timeout() bool { return true } +func (http_tlsHandshakeTimeoutError) Temporary() bool { return true } -func (tlsHandshakeTimeoutError) Temporary() bool { return true } - -func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } - -type noteEOFReader struct { - r io.Reader - sawEOF *bool -} - -func (nr noteEOFReader) Read(p []byte) (n int, err error) { - n, err = nr.r.Read(p) - if err == io.EOF { - *nr.sawEOF = true - } - return -} +func (http_tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } // fakeLocker is a sync.Locker which does nothing. It's used to guard // test-only fields when not under test, to avoid runtime atomic // overhead. -type fakeLocker struct{} +type http_fakeLocker struct{} -func (fakeLocker) Lock() {} +func (http_fakeLocker) Lock() {} -func (fakeLocker) Unlock() {} +func (http_fakeLocker) Unlock() {} -func isNetWriteError(err error) bool { - switch e := err.(type) { - case *url.Error: - return isNetWriteError(e.Err) - case *net.OpError: - return e.Op == "write" - default: - return false - } -} - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. +// cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if +// cfg is nil. This is safe to call even if cfg is in active use by a TLS +// client or server. // -// The cfg must not be in active use by tls.Server, or else -// there can still be a race with tls.Server updating SessionTicketKey -// and our copying it, and also a race with the server setting -// SessionTicketsDisabled=false on failure to set the random -// ticket key. +// cloneTLSConfig should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/searKing/golang // -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname cloneTLSConfig +func http_cloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, + return cfg.Clone() +} + +type http_connLRU struct { + ll *list.List // list.Element.Value type is of *persistConn + m map[*http_persistConn]*list.Element +} + +// add adds pc to the head of the linked list. +func (cl *http_connLRU) add(pc *http_persistConn) { + if cl.ll == nil { + cl.ll = list.New() + cl.m = make(map[*http_persistConn]*list.Element) + } + ele := cl.ll.PushFront(pc) + if _, ok := cl.m[pc]; ok { + panic("persistConn was already in LRU") + } + cl.m[pc] = ele +} + +func (cl *http_connLRU) removeOldest() *http_persistConn { + ele := cl.ll.Back() + pc := ele.Value.(*http_persistConn) + cl.ll.Remove(ele) + delete(cl.m, pc) + return pc +} + +// remove removes pc from cl. +func (cl *http_connLRU) remove(pc *http_persistConn) { + if ele, ok := cl.m[pc]; ok { + cl.ll.Remove(ele) + delete(cl.m, pc) } } -// cloneTLSClientConfig is like cloneTLSConfig but omits -// the fields SessionTicketsDisabled and SessionTicketKey. -// This makes it safe to call cloneTLSClientConfig on a config -// in active use by a server. -func cloneTLSClientConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - } +// len returns the number of items in the cache. +func (cl *http_connLRU) len() int { + return len(cl.m) +} + +func http_defaultTransportDialContext(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) { + return dialer.DialContext } `
diff --git a/go.mod b/go.mod index be31c6b..1d5b6f2 100644 --- a/go.mod +++ b/go.mod
@@ -7,13 +7,13 @@ github.com/biogo/biogo v1.0.4 github.com/biogo/graph v0.0.0-20150317020928-057c1989faed github.com/biogo/store v0.0.0-20201120204734-aad293a2328f - github.com/blevesearch/bleve v1.0.14 + github.com/blevesearch/bleve/v2 v2.4.4 github.com/dustin/go-wikiparse v0.0.0-20211018054215-c01ec186f20c github.com/gomodule/redigo v1.9.2 github.com/google/pprof v0.0.0-20241017200806-017d972448fc github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/opencontainers/runtime-spec v1.0.2 - github.com/yuin/gopher-lua v1.1.1 + github.com/yuin/gopher-lua v1.1.2 gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a go.chromium.org/luci v0.0.0-20250606083430-318ed8f4881d go.etcd.io/etcd/client/v3 v3.5.8 @@ -23,34 +23,39 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect - github.com/RoaringBitmap/roaring v0.4.23 // indirect + github.com/RoaringBitmap/roaring v1.9.3 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect + github.com/blevesearch/bleve_index_api v1.1.12 // indirect + github.com/blevesearch/geo v0.1.20 // indirect + github.com/blevesearch/go-faiss v1.0.24 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect - github.com/blevesearch/mmap-go v1.0.2 // indirect - github.com/blevesearch/segment v0.9.0 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.16 // indirect + github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect - github.com/blevesearch/zap/v11 v11.0.14 // indirect - github.com/blevesearch/zap/v12 v12.0.14 // indirect - github.com/blevesearch/zap/v13 v13.0.6 // indirect - github.com/blevesearch/zap/v14 v14.0.5 // indirect - github.com/blevesearch/zap/v15 v15.0.3 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.16 // indirect + github.com/blevesearch/zapx/v16 v16.1.9-0.20241217210638-a0519e7caf3b // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/couchbase/vellum v1.0.2 // indirect github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 // indirect - github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/philhofer/fwd v1.0.0 // indirect - github.com/steveyen/gtreap v0.1.0 // indirect github.com/stretchr/testify v1.10.0 // indirect - github.com/tinylib/msgp v1.1.0 // indirect - github.com/willf/bitset v1.1.10 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 // indirect gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 // indirect
diff --git a/go.sum b/go.sum index b74134f..b5c5b79 100644 --- a/go.sum +++ b/go.sum
@@ -1,13 +1,11 @@ cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= +github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/biogo/biogo v1.0.4 h1:I+FV8WHty5o6pk1VWZxwFETJDcd25GKcGsghMTeQgCY= @@ -19,49 +17,52 @@ github.com/biogo/store v0.0.0-20200104231603-2c6ad937eb83/go.mod h1:wdbXg77soR6ESRprAMEwAQDFtLT6EAGF5o1GRy0cB5k= github.com/biogo/store v0.0.0-20201120204734-aad293a2328f h1:+6okTAeUsUrdQr/qN7fIODzowrjjCrnJDg/gkYqcSXY= github.com/biogo/store v0.0.0-20201120204734-aad293a2328f/go.mod h1:z52shMwD6SGwRg2iYFjjDwX5Ene4ENTw6HfXraUy/08= -github.com/blevesearch/bleve v1.0.14 h1:Q8r+fHTt35jtGXJUM0ULwM3Tzg+MRfyai4ZkWDy2xO4= -github.com/blevesearch/bleve v1.0.14/go.mod h1:e/LJTr+E7EaoVdkQZTfoz7dt4KoDNvDbLb8MSKuNTLQ= -github.com/blevesearch/blevex v1.0.0 h1:pnilj2Qi3YSEGdWgLj1Pn9Io7ukfXPoQcpAI1Bv8n/o= -github.com/blevesearch/blevex v1.0.0/go.mod h1:2rNVqoG2BZI8t1/P1awgTKnGlx5MP9ZbtEciQaNhswc= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blevesearch/bleve/v2 v2.4.4 h1:RwwLGjUm54SwyyykbrZs4vc1qjzYic4ZnAnY9TwNl60= +github.com/blevesearch/bleve/v2 v2.4.4/go.mod h1:fa2Eo6DP7JR+dMFpQe+WiZXINKSunh7WBtlDGbolKXk= +github.com/blevesearch/bleve_index_api v1.1.12 h1:P4bw9/G/5rulOF7SJ9l4FsDoo7UFJ+5kexNy1RXfegY= +github.com/blevesearch/bleve_index_api v1.1.12/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.24 h1:K79IvKjoKHdi7FdiXEsAhxpMuns0x4fM0BO93bW5jLI= +github.com/blevesearch/go-faiss v1.0.24/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= -github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.2.16 h1:uGvKVvG7zvSxCwcm4/ehBa9cCEuZVE+/zvrSl57QUVY= +github.com/blevesearch/scorch_segment_api/v2 v2.2.16/go.mod h1:VF5oHVbIFTu+znY1v30GjSpT5+9YFs9dV2hjvuh34F0= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/zap/v11 v11.0.14 h1:IrDAvtlzDylh6H2QCmS0OGcN9Hpf6mISJlfKjcwJs7k= -github.com/blevesearch/zap/v11 v11.0.14/go.mod h1:MUEZh6VHGXv1PKx3WnCbdP404LGG2IZVa/L66pyFwnY= -github.com/blevesearch/zap/v12 v12.0.14 h1:2o9iRtl1xaRjsJ1xcqTyLX414qPAwykHNV7wNVmbp3w= -github.com/blevesearch/zap/v12 v12.0.14/go.mod h1:rOnuZOiMKPQj18AEKEHJxuI14236tTQ1ZJz4PAnWlUg= -github.com/blevesearch/zap/v13 v13.0.6 h1:r+VNSVImi9cBhTNNR+Kfl5uiGy8kIbb0JMz/h8r6+O4= -github.com/blevesearch/zap/v13 v13.0.6/go.mod h1:L89gsjdRKGyGrRN6nCpIScCvvkyxvmeDCwZRcjjPCrw= -github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67njR0NU= -github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY= -github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY= -github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.16 h1:Ct3rv7FUJPfPk99TI/OofdC+Kpb4IdyfdMH48sb+FmE= +github.com/blevesearch/zapx/v15 v15.3.16/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v16 v16.1.9-0.20241217210638-a0519e7caf3b h1:ju9Az5YgrzCeK3M1QwvZIpxYhChkXp7/L0RhDYsxXoE= +github.com/blevesearch/zapx/v16 v16.1.9-0.20241217210638-a0519e7caf3b/go.mod h1:BlrYNpOu4BvVRslmIG+rLtKhmjIaRhIbG8sb9scGTwI= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/go-couchbase v0.1.0/go.mod h1:+/bddYDxXsf9qt0xpDUtRR47A2GjaXmGGAqQ/k3GJ8A= github.com/couchbase/gomemcached v0.1.3/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= github.com/couchbase/goutils v0.1.1/go.mod h1:h89Ek/tiOxxqjz30nPPlwZdQbdB8BwgnuBxeoUe/ViE= -github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= -github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= -github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 h1:m+Fkk9QEMuV6Z1ithqqYogOHV7Pl6rMKe34NBTJTS/c= github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22/go.mod h1:jXqs4TJbb7Xtl0FwUgBaOXty8edb/61H37U4D9E5EQE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -73,14 +74,6 @@ github.com/dustin/go-wikiparse v0.0.0-20211018054215-c01ec186f20c h1:2LZhS221ttVN6g2xi9u5iTSHs9kZ+B0qkAvpsHlifMc= github.com/dustin/go-wikiparse v0.0.0-20211018054215-c01ec186f20c/go.mod h1:BWPhJUMpzRU9dWxw1V78ZKsZgOGSIz8e0bybAoSNoPY= github.com/dustin/httputil v0.0.0-20170305193905-c47743f54f89/go.mod h1:ZoDWdnxro8Kesk3zrCNOHNFWtajFPSnDMjVEjGjQu/0= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -91,34 +84,26 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241017200806-017d972448fc h1:NGyrhhFhwvRAZg02jnYVg3GBQy0qGBKmFQJwaPmpmxs= github.com/google/pprof v0.0.0-20241017200806-017d972448fc/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede h1:YrgBGwxMRK0Vq0WSCWFaZUnTsrA/PZE/xs1QZh+/edg= +github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -127,7 +112,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/kortschak/utter v0.0.0-20190412033250-50fe362e6560/go.mod h1:oDr41C7kH9wvAikWyFhr6UFr8R7nelpmCF5XR5rL7I8= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -137,74 +121,40 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= -github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= -github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= +github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 h1:K+bMSIx9A7mLES1rtG+qKduLIXq40DAzYHtb0XuCukA= gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181/go.mod h1:dzYhVIwWCtzPAa4QP98wfB9+mzt33MSmM8wsKiMi2ow= gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 h1:oYrL81N608MLZhma3ruL8qTM4xcpYECGut8KSxRY59g= @@ -219,7 +169,6 @@ gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU= go.chromium.org/luci v0.0.0-20250606083430-318ed8f4881d h1:Fe3QkOf14gQnO2qNRfy4lh4hWeE7KfOR9BnbUrnT944= go.chromium.org/luci v0.0.0-20250606083430-318ed8f4881d/go.mod h1:Sq6fyDyzMSLbIcXyceGDn9IZV0MW/DoidMCUOBa1/+M= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.8 h1:Zf44zJszoU7zRV0X/nStPenegNXoFDWcB/MwrJbA+L4= @@ -248,7 +197,6 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -267,7 +215,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -278,26 +225,21 @@ golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= @@ -333,12 +275,9 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/sweet/benchmarks/bleve-index/main.go b/sweet/benchmarks/bleve-index/main.go index 9724bbd..d94d48d 100644 --- a/sweet/benchmarks/bleve-index/main.go +++ b/sweet/benchmarks/bleve-index/main.go
@@ -16,8 +16,8 @@ "golang.org/x/benchmarks/sweet/benchmarks/internal/driver" blevebench "golang.org/x/benchmarks/third_party/bleve-bench" - "github.com/blevesearch/bleve" - _ "github.com/blevesearch/bleve/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2" + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" wikiparse "github.com/dustin/go-wikiparse" )
diff --git a/third_party/bleve-bench/mapping.go b/third_party/bleve-bench/mapping.go index a3421f1..d47f030 100644 --- a/third_party/bleve-bench/mapping.go +++ b/third_party/bleve-bench/mapping.go
@@ -21,8 +21,8 @@ package blevebench import ( - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" ) // ArticleMapping returns a mapping for indexing wikipedia articles