| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package http |
| |
| import ( |
| "bufio" |
| "bytes" |
| "errors" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "net/http/internal" |
| "net/textproto" |
| "sort" |
| "strconv" |
| "strings" |
| "sync" |
| ) |
| |
| // ErrLineTooLong is returned when reading request or response bodies |
| // with malformed chunked encoding. |
| var ErrLineTooLong = internal.ErrLineTooLong |
| |
| type errorReader struct { |
| err error |
| } |
| |
| func (r *errorReader) Read(p []byte) (n int, err error) { |
| return 0, r.err |
| } |
| |
| // transferWriter inspects the fields of a user-supplied Request or Response, |
| // sanitizes them without changing the user object and provides methods for |
| // writing the respective header, body and trailer in wire format. |
| type transferWriter struct { |
| Method string |
| Body io.Reader |
| BodyCloser io.Closer |
| ResponseToHEAD bool |
| ContentLength int64 // -1 means unknown, 0 means exactly none |
| Close bool |
| TransferEncoding []string |
| Trailer Header |
| } |
| |
| func newTransferWriter(r interface{}) (t *transferWriter, err error) { |
| t = &transferWriter{} |
| |
| // Extract relevant fields |
| atLeastHTTP11 := false |
| switch rr := r.(type) { |
| case *Request: |
| if rr.ContentLength != 0 && rr.Body == nil { |
| return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) |
| } |
| t.Method = rr.Method |
| t.Body = rr.Body |
| t.BodyCloser = rr.Body |
| t.ContentLength = rr.ContentLength |
| t.Close = rr.Close |
| t.TransferEncoding = rr.TransferEncoding |
| t.Trailer = rr.Trailer |
| atLeastHTTP11 = rr.ProtoAtLeast(1, 1) |
| if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 { |
| if t.ContentLength == 0 { |
| // Test to see if it's actually zero or just unset. |
| var buf [1]byte |
| n, rerr := io.ReadFull(t.Body, buf[:]) |
| if rerr != nil && rerr != io.EOF { |
| t.ContentLength = -1 |
| t.Body = &errorReader{rerr} |
| } else if n == 1 { |
| // Oh, guess there is data in this Body Reader after all. |
| // The ContentLength field just wasn't set. |
| // Stich the Body back together again, re-attaching our |
| // consumed byte. |
| t.ContentLength = -1 |
| t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body) |
| } else { |
| // Body is actually empty. |
| t.Body = nil |
| t.BodyCloser = nil |
| } |
| } |
| if t.ContentLength < 0 { |
| t.TransferEncoding = []string{"chunked"} |
| } |
| } |
| case *Response: |
| if rr.Request != nil { |
| t.Method = rr.Request.Method |
| } |
| t.Body = rr.Body |
| t.BodyCloser = rr.Body |
| t.ContentLength = rr.ContentLength |
| t.Close = rr.Close |
| t.TransferEncoding = rr.TransferEncoding |
| t.Trailer = rr.Trailer |
| atLeastHTTP11 = rr.ProtoAtLeast(1, 1) |
| t.ResponseToHEAD = noBodyExpected(t.Method) |
| } |
| |
| // Sanitize Body,ContentLength,TransferEncoding |
| if t.ResponseToHEAD { |
| t.Body = nil |
| if chunked(t.TransferEncoding) { |
| t.ContentLength = -1 |
| } |
| } else { |
| if !atLeastHTTP11 || t.Body == nil { |
| t.TransferEncoding = nil |
| } |
| if chunked(t.TransferEncoding) { |
| t.ContentLength = -1 |
| } else if t.Body == nil { // no chunking, no body |
| t.ContentLength = 0 |
| } |
| } |
| |
| // Sanitize Trailer |
| if !chunked(t.TransferEncoding) { |
| t.Trailer = nil |
| } |
| |
| return t, nil |
| } |
| |
| func noBodyExpected(requestMethod string) bool { |
| return requestMethod == "HEAD" |
| } |
| |
| func (t *transferWriter) shouldSendContentLength() bool { |
| if chunked(t.TransferEncoding) { |
| return false |
| } |
| if t.ContentLength > 0 { |
| return true |
| } |
| // Many servers expect a Content-Length for these methods |
| if t.Method == "POST" || t.Method == "PUT" { |
| return true |
| } |
| if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { |
| return true |
| } |
| |
| return false |
| } |
| |
| func (t *transferWriter) WriteHeader(w io.Writer) error { |
| if t.Close { |
| if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { |
| return err |
| } |
| } |
| |
| // Write Content-Length and/or Transfer-Encoding whose values are a |
| // function of the sanitized field triple (Body, ContentLength, |
| // TransferEncoding) |
| if t.shouldSendContentLength() { |
| if _, err := io.WriteString(w, "Content-Length: "); err != nil { |
| return err |
| } |
| if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { |
| return err |
| } |
| } else if chunked(t.TransferEncoding) { |
| if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { |
| return err |
| } |
| } |
| |
| // Write Trailer header |
| if t.Trailer != nil { |
| keys := make([]string, 0, len(t.Trailer)) |
| for k := range t.Trailer { |
| k = CanonicalHeaderKey(k) |
| switch k { |
| case "Transfer-Encoding", "Trailer", "Content-Length": |
| return &badStringError{"invalid Trailer key", k} |
| } |
| keys = append(keys, k) |
| } |
| if len(keys) > 0 { |
| sort.Strings(keys) |
| // TODO: could do better allocation-wise here, but trailers are rare, |
| // so being lazy for now. |
| if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { |
| return err |
| } |
| } |
| } |
| |
| return nil |
| } |
| |
| func (t *transferWriter) WriteBody(w io.Writer) error { |
| var err error |
| var ncopy int64 |
| |
| // Write body |
| if t.Body != nil { |
| if chunked(t.TransferEncoding) { |
| cw := internal.NewChunkedWriter(w) |
| _, err = io.Copy(cw, t.Body) |
| if err == nil { |
| err = cw.Close() |
| } |
| } else if t.ContentLength == -1 { |
| ncopy, err = io.Copy(w, t.Body) |
| } else { |
| ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) |
| if err != nil { |
| return err |
| } |
| var nextra int64 |
| nextra, err = io.Copy(ioutil.Discard, t.Body) |
| ncopy += nextra |
| } |
| if err != nil { |
| return err |
| } |
| if err = t.BodyCloser.Close(); err != nil { |
| return err |
| } |
| } |
| |
| if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { |
| return fmt.Errorf("http: ContentLength=%d with Body length %d", |
| t.ContentLength, ncopy) |
| } |
| |
| if chunked(t.TransferEncoding) { |
| // Write Trailer header |
| if t.Trailer != nil { |
| if err := t.Trailer.Write(w); err != nil { |
| return err |
| } |
| } |
| // Last chunk, empty trailer |
| _, err = io.WriteString(w, "\r\n") |
| } |
| return err |
| } |
| |
| type transferReader struct { |
| // Input |
| Header Header |
| StatusCode int |
| RequestMethod string |
| ProtoMajor int |
| ProtoMinor int |
| // Output |
| Body io.ReadCloser |
| ContentLength int64 |
| TransferEncoding []string |
| Close bool |
| Trailer Header |
| } |
| |
| // bodyAllowedForStatus reports whether a given response status code |
| // permits a body. See RFC2616, section 4.4. |
| func bodyAllowedForStatus(status int) bool { |
| switch { |
| case status >= 100 && status <= 199: |
| return false |
| case status == 204: |
| return false |
| case status == 304: |
| return false |
| } |
| return true |
| } |
| |
| var ( |
| suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} |
| suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} |
| ) |
| |
| func suppressedHeaders(status int) []string { |
| switch { |
| case status == 304: |
| // RFC 2616 section 10.3.5: "the response MUST NOT include other entity-headers" |
| return suppressedHeaders304 |
| case !bodyAllowedForStatus(status): |
| return suppressedHeadersNoBody |
| } |
| return nil |
| } |
| |
| // msg is *Request or *Response. |
| func readTransfer(msg interface{}, r *bufio.Reader) (err error) { |
| t := &transferReader{RequestMethod: "GET"} |
| |
| // Unify input |
| isResponse := false |
| switch rr := msg.(type) { |
| case *Response: |
| t.Header = rr.Header |
| t.StatusCode = rr.StatusCode |
| t.ProtoMajor = rr.ProtoMajor |
| t.ProtoMinor = rr.ProtoMinor |
| t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) |
| isResponse = true |
| if rr.Request != nil { |
| t.RequestMethod = rr.Request.Method |
| } |
| case *Request: |
| t.Header = rr.Header |
| t.ProtoMajor = rr.ProtoMajor |
| t.ProtoMinor = rr.ProtoMinor |
| // Transfer semantics for Requests are exactly like those for |
| // Responses with status code 200, responding to a GET method |
| t.StatusCode = 200 |
| default: |
| panic("unexpected type") |
| } |
| |
| // Default to HTTP/1.1 |
| if t.ProtoMajor == 0 && t.ProtoMinor == 0 { |
| t.ProtoMajor, t.ProtoMinor = 1, 1 |
| } |
| |
| // Transfer encoding, content length |
| t.TransferEncoding, err = fixTransferEncoding(t.RequestMethod, t.Header) |
| if err != nil { |
| return err |
| } |
| |
| realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding) |
| if err != nil { |
| return err |
| } |
| if isResponse && t.RequestMethod == "HEAD" { |
| if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { |
| return err |
| } else { |
| t.ContentLength = n |
| } |
| } else { |
| t.ContentLength = realLength |
| } |
| |
| // Trailer |
| t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding) |
| if err != nil { |
| return err |
| } |
| |
| // If there is no Content-Length or chunked Transfer-Encoding on a *Response |
| // and the status is not 1xx, 204 or 304, then the body is unbounded. |
| // See RFC2616, section 4.4. |
| switch msg.(type) { |
| case *Response: |
| if realLength == -1 && |
| !chunked(t.TransferEncoding) && |
| bodyAllowedForStatus(t.StatusCode) { |
| // Unbounded body. |
| t.Close = true |
| } |
| } |
| |
| // Prepare body reader. ContentLength < 0 means chunked encoding |
| // or close connection when finished, since multipart is not supported yet |
| switch { |
| case chunked(t.TransferEncoding): |
| if noBodyExpected(t.RequestMethod) { |
| t.Body = eofReader |
| } else { |
| t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} |
| } |
| case realLength == 0: |
| t.Body = eofReader |
| case realLength > 0: |
| t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} |
| default: |
| // realLength < 0, i.e. "Content-Length" not mentioned in header |
| if t.Close { |
| // Close semantics (i.e. HTTP/1.0) |
| t.Body = &body{src: r, closing: t.Close} |
| } else { |
| // Persistent connection (i.e. HTTP/1.1) |
| t.Body = eofReader |
| } |
| } |
| |
| // Unify output |
| switch rr := msg.(type) { |
| case *Request: |
| rr.Body = t.Body |
| rr.ContentLength = t.ContentLength |
| rr.TransferEncoding = t.TransferEncoding |
| rr.Close = t.Close |
| rr.Trailer = t.Trailer |
| case *Response: |
| rr.Body = t.Body |
| rr.ContentLength = t.ContentLength |
| rr.TransferEncoding = t.TransferEncoding |
| rr.Close = t.Close |
| rr.Trailer = t.Trailer |
| } |
| |
| return nil |
| } |
| |
| // Checks whether chunked is part of the encodings stack |
| func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } |
| |
| // Checks whether the encoding is explicitly "identity". |
| func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } |
| |
| // Sanitize transfer encoding |
| func fixTransferEncoding(requestMethod string, header Header) ([]string, error) { |
| raw, present := header["Transfer-Encoding"] |
| if !present { |
| return nil, nil |
| } |
| |
| delete(header, "Transfer-Encoding") |
| |
| encodings := strings.Split(raw[0], ",") |
| te := make([]string, 0, len(encodings)) |
| // TODO: Even though we only support "identity" and "chunked" |
| // encodings, the loop below is designed with foresight. One |
| // invariant that must be maintained is that, if present, |
| // chunked encoding must always come first. |
| for _, encoding := range encodings { |
| encoding = strings.ToLower(strings.TrimSpace(encoding)) |
| // "identity" encoding is not recorded |
| if encoding == "identity" { |
| break |
| } |
| if encoding != "chunked" { |
| return nil, &badStringError{"unsupported transfer encoding", encoding} |
| } |
| te = te[0 : len(te)+1] |
| te[len(te)-1] = encoding |
| } |
| if len(te) > 1 { |
| return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")} |
| } |
| if len(te) > 0 { |
| // Chunked encoding trumps Content-Length. See RFC 2616 |
| // Section 4.4. Currently len(te) > 0 implies chunked |
| // encoding. |
| delete(header, "Content-Length") |
| return te, nil |
| } |
| |
| return nil, nil |
| } |
| |
| // Determine the expected body length, using RFC 2616 Section 4.4. This |
| // function is not a method, because ultimately it should be shared by |
| // ReadResponse and ReadRequest. |
| func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { |
| |
| // Logic based on response type or status |
| if noBodyExpected(requestMethod) { |
| return 0, nil |
| } |
| if status/100 == 1 { |
| return 0, nil |
| } |
| switch status { |
| case 204, 304: |
| return 0, nil |
| } |
| |
| // Logic based on Transfer-Encoding |
| if chunked(te) { |
| return -1, nil |
| } |
| |
| // Logic based on Content-Length |
| cl := strings.TrimSpace(header.get("Content-Length")) |
| if cl != "" { |
| n, err := parseContentLength(cl) |
| if err != nil { |
| return -1, err |
| } |
| return n, nil |
| } else { |
| header.Del("Content-Length") |
| } |
| |
| if !isResponse && requestMethod == "GET" { |
| // RFC 2616 doesn't explicitly permit nor forbid an |
| // entity-body on a GET request so we permit one if |
| // declared, but we default to 0 here (not -1 below) |
| // if there's no mention of a body. |
| return 0, nil |
| } |
| |
| // Body-EOF logic based on other methods (like closing, or chunked coding) |
| return -1, nil |
| } |
| |
| // Determine whether to hang up after sending a request and body, or |
| // receiving a response and body |
| // 'header' is the request headers |
| func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { |
| if major < 1 { |
| return true |
| } else if major == 1 && minor == 0 { |
| if !strings.Contains(strings.ToLower(header.get("Connection")), "keep-alive") { |
| return true |
| } |
| return false |
| } else { |
| // TODO: Should split on commas, toss surrounding white space, |
| // and check each field. |
| if strings.ToLower(header.get("Connection")) == "close" { |
| if removeCloseHeader { |
| header.Del("Connection") |
| } |
| return true |
| } |
| } |
| return false |
| } |
| |
| // Parse the trailer header |
| func fixTrailer(header Header, te []string) (Header, error) { |
| raw := header.get("Trailer") |
| if raw == "" { |
| return nil, nil |
| } |
| |
| header.Del("Trailer") |
| trailer := make(Header) |
| keys := strings.Split(raw, ",") |
| for _, key := range keys { |
| key = CanonicalHeaderKey(strings.TrimSpace(key)) |
| switch key { |
| case "Transfer-Encoding", "Trailer", "Content-Length": |
| return nil, &badStringError{"bad trailer key", key} |
| } |
| trailer[key] = nil |
| } |
| if len(trailer) == 0 { |
| return nil, nil |
| } |
| if !chunked(te) { |
| // Trailer and no chunking |
| return nil, ErrUnexpectedTrailer |
| } |
| return trailer, nil |
| } |
| |
| // body turns a Reader into a ReadCloser. |
| // Close ensures that the body has been fully read |
| // and then reads the trailer if necessary. |
| type body struct { |
| src io.Reader |
| hdr interface{} // non-nil (Response or Request) value means read trailer |
| r *bufio.Reader // underlying wire-format reader for the trailer |
| closing bool // is the connection to be closed after reading body? |
| |
| mu sync.Mutex // guards closed, and calls to Read and Close |
| closed bool |
| } |
| |
| // ErrBodyReadAfterClose is returned when reading a Request or Response |
| // Body after the body has been closed. This typically happens when the body is |
| // read after an HTTP Handler calls WriteHeader or Write on its |
| // ResponseWriter. |
| var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") |
| |
| func (b *body) Read(p []byte) (n int, err error) { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| if b.closed { |
| return 0, ErrBodyReadAfterClose |
| } |
| return b.readLocked(p) |
| } |
| |
| // Must hold b.mu. |
| func (b *body) readLocked(p []byte) (n int, err error) { |
| n, err = b.src.Read(p) |
| |
| if err == io.EOF { |
| // Chunked case. Read the trailer. |
| if b.hdr != nil { |
| if e := b.readTrailer(); e != nil { |
| err = e |
| } |
| b.hdr = nil |
| } else { |
| // If the server declared the Content-Length, our body is a LimitedReader |
| // and we need to check whether this EOF arrived early. |
| if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { |
| err = io.ErrUnexpectedEOF |
| } |
| } |
| } |
| |
| // If we can return an EOF here along with the read data, do |
| // so. This is optional per the io.Reader contract, but doing |
| // so helps the HTTP transport code recycle its connection |
| // earlier (since it will see this EOF itself), even if the |
| // client doesn't do future reads or Close. |
| if err == nil && n > 0 { |
| if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { |
| err = io.EOF |
| } |
| } |
| |
| return n, err |
| } |
| |
| var ( |
| singleCRLF = []byte("\r\n") |
| doubleCRLF = []byte("\r\n\r\n") |
| ) |
| |
| func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { |
| for peekSize := 4; ; peekSize++ { |
| // This loop stops when Peek returns an error, |
| // which it does when r's buffer has been filled. |
| buf, err := r.Peek(peekSize) |
| if bytes.HasSuffix(buf, doubleCRLF) { |
| return true |
| } |
| if err != nil { |
| break |
| } |
| } |
| return false |
| } |
| |
| var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") |
| |
| func (b *body) readTrailer() error { |
| // The common case, since nobody uses trailers. |
| buf, err := b.r.Peek(2) |
| if bytes.Equal(buf, singleCRLF) { |
| b.r.Discard(2) |
| return nil |
| } |
| if len(buf) < 2 { |
| return errTrailerEOF |
| } |
| if err != nil { |
| return err |
| } |
| |
| // Make sure there's a header terminator coming up, to prevent |
| // a DoS with an unbounded size Trailer. It's not easy to |
| // slip in a LimitReader here, as textproto.NewReader requires |
| // a concrete *bufio.Reader. Also, we can't get all the way |
| // back up to our conn's LimitedReader that *might* be backing |
| // this bufio.Reader. Instead, a hack: we iteratively Peek up |
| // to the bufio.Reader's max size, looking for a double CRLF. |
| // This limits the trailer to the underlying buffer size, typically 4kB. |
| if !seeUpcomingDoubleCRLF(b.r) { |
| return errors.New("http: suspiciously long trailer after chunked body") |
| } |
| |
| hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() |
| if err != nil { |
| if err == io.EOF { |
| return errTrailerEOF |
| } |
| return err |
| } |
| switch rr := b.hdr.(type) { |
| case *Request: |
| mergeSetHeader(&rr.Trailer, Header(hdr)) |
| case *Response: |
| mergeSetHeader(&rr.Trailer, Header(hdr)) |
| } |
| return nil |
| } |
| |
| func mergeSetHeader(dst *Header, src Header) { |
| if *dst == nil { |
| *dst = src |
| return |
| } |
| for k, vv := range src { |
| (*dst)[k] = vv |
| } |
| } |
| |
| func (b *body) Close() error { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| if b.closed { |
| return nil |
| } |
| var err error |
| switch { |
| case b.hdr == nil && b.closing: |
| // no trailer and closing the connection next. |
| // no point in reading to EOF. |
| default: |
| // Fully consume the body, which will also lead to us reading |
| // the trailer headers after the body, if present. |
| _, err = io.Copy(ioutil.Discard, bodyLocked{b}) |
| } |
| b.closed = true |
| return err |
| } |
| |
| // bodyLocked is a io.Reader reading from a *body when its mutex is |
| // already held. |
| type bodyLocked struct { |
| b *body |
| } |
| |
| func (bl bodyLocked) Read(p []byte) (n int, err error) { |
| if bl.b.closed { |
| return 0, ErrBodyReadAfterClose |
| } |
| return bl.b.readLocked(p) |
| } |
| |
| // parseContentLength trims whitespace from s and returns -1 if no value |
| // is set, or the value if it's >= 0. |
| func parseContentLength(cl string) (int64, error) { |
| cl = strings.TrimSpace(cl) |
| if cl == "" { |
| return -1, nil |
| } |
| n, err := strconv.ParseInt(cl, 10, 64) |
| if err != nil || n < 0 { |
| return 0, &badStringError{"bad Content-Length", cl} |
| } |
| return n, nil |
| |
| } |